diff --git a/scripts/audio-sender-sine-tester.js b/scripts/audio-sender-sine-tester.js
new file mode 100644
index 0000000..b6f6bed
--- /dev/null
+++ b/scripts/audio-sender-sine-tester.js
@@ -0,0 +1,371 @@
+// ==UserScript==
+// @name 12345 Audio Sender Sine Tester
+// @namespace http://tampermonkey.net/
+// @version 1.0
+// @description List current WebRTC audio senders and temporarily send a sine wave through a selected sender.
+// @author You
+// @match *://yhy-yw.22sj.cn:*/*
+// @grant none
+// @run-at document-start
+// ==/UserScript==
+
+(function () {
+ 'use strict';
+
+ const originalAddTrack = RTCPeerConnection.prototype.addTrack;
+ const originalAddTransceiver = RTCPeerConnection.prototype.addTransceiver;
+ const originalSetRemoteDescription = RTCPeerConnection.prototype.setRemoteDescription;
+
+ const senderCandidates = new Map();
+ const outboundStatsMemory = new Map();
+
+ let activeTest = null;
+ let lastRenderAt = 0;
+
+ function nowText() {
+ return new Date().toLocaleTimeString([], { hour12: false });
+ }
+
+ function shortId(id) {
+ if (!id) return 'null';
+ return id.length <= 18 ? id : `${id.slice(0, 8)}...${id.slice(-6)}`;
+ }
+
+ function log(message) {
+ console.log(`[Audio Sine Tester] ${message}`);
+ const box = document.getElementById('audio-sine-tester-log');
+ if (!box) return;
+ const row = document.createElement('div');
+ row.textContent = `[${nowText()}] ${message}`;
+ box.appendChild(row);
+ while (box.children.length > 80) box.removeChild(box.firstChild);
+ box.scrollTop = box.scrollHeight;
+ }
+
+ function ensureSenderCandidate(sender, track, source) {
+ if (!sender) return null;
+ const senderTrack = track || sender.track;
+ if (senderTrack && senderTrack.kind !== 'audio') return null;
+
+ const id = sender.__audioSineTesterId || `sender-${senderCandidates.size + 1}`;
+ sender.__audioSineTesterId = id;
+
+ let candidate = senderCandidates.get(id);
+ if (!candidate) {
+ candidate = {
+ id,
+ sender,
+ track: senderTrack || null,
+ originalTrack: senderTrack || null,
+ sources: new Set(),
+ bytesSent: 0,
+ packetsSent: 0,
+ bytesDelta: 0,
+ packetsDelta: 0,
+ score: 0,
+ };
+ senderCandidates.set(id, candidate);
+ log(`found audio sender ${id} (${source}) track=${senderTrack?.id || 'null'}`);
+ }
+
+ candidate.sender = sender;
+ candidate.track = sender.track || senderTrack || candidate.track;
+ if (sender.track && sender.track.kind === 'audio' && !sender.track.__audioSineTesterTrack) {
+ candidate.originalTrack = sender.track;
+ }
+ candidate.sources.add(source);
+ wrapSender(sender);
+ return candidate;
+ }
+
+ function wrapSender(sender) {
+ if (!sender || sender.__audioSineTesterWrapped) return;
+ const originalReplaceTrack = sender.replaceTrack?.bind(sender);
+ if (!originalReplaceTrack) return;
+
+ sender.replaceTrack = async function (track) {
+ const result = await originalReplaceTrack(track);
+ const candidate = ensureSenderCandidate(sender, track, 'replaceTrack');
+ if (candidate) {
+ candidate.track = track || sender.track || candidate.track;
+ if (track && !track.__audioSineTesterTrack) candidate.originalTrack = track;
+ log(`${candidate.id} replaceTrack -> ${track?.id || 'null'}`);
+ }
+ return result;
+ };
+ sender.__audioSineTesterWrapped = true;
+ }
+
+ async function refreshSenderStats() {
+ for (const candidate of senderCandidates.values()) {
+ const sender = candidate.sender;
+ candidate.track = sender.track || candidate.track;
+ if (!sender.getStats) continue;
+
+ try {
+ const report = await sender.getStats();
+ report.forEach((stat) => {
+ const kind = stat.kind || stat.mediaType;
+ if (stat.type !== 'outbound-rtp' || kind !== 'audio') return;
+
+ const key = `${candidate.id}:${stat.id}`;
+ const prev = outboundStatsMemory.get(key);
+ const bytes = stat.bytesSent || 0;
+ const packets = stat.packetsSent || 0;
+ candidate.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0;
+ candidate.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0;
+ candidate.bytesSent = bytes;
+ candidate.packetsSent = packets;
+ outboundStatsMemory.set(key, { bytes, packets });
+ candidate.score = scoreCandidate(candidate);
+ });
+ } catch (error) {
+ candidate.sources.add(`stats-error:${error.message || error}`);
+ }
+ }
+ }
+
+ function scoreCandidate(candidate) {
+ let score = 0;
+ if (candidate.track?.kind === 'audio') score += 1000;
+ if (candidate.track?.readyState === 'live') score += 800;
+ if (candidate.sources.has('pc-addTrack')) score += 1500;
+ if (candidate.bytesSent > 0) score += 600;
+ if (candidate.bytesDelta > 0) score += Math.min(2500, candidate.bytesDelta * 4);
+ if (candidate.packetsDelta > 0) score += Math.min(1000, candidate.packetsDelta * 12);
+ return score;
+ }
+
+ function makeSineTrack(frequency) {
+ const AudioCtx = window.AudioContext || window.webkitAudioContext;
+ const context = new AudioCtx({ sampleRate: 48000 });
+ const oscillator = context.createOscillator();
+ const gain = context.createGain();
+ const dest = context.createMediaStreamDestination();
+
+ oscillator.type = 'sine';
+ oscillator.frequency.value = frequency;
+ gain.gain.value = 0.08;
+ oscillator.connect(gain);
+ gain.connect(dest);
+ oscillator.start();
+
+ const track = dest.stream.getAudioTracks()[0];
+ track.__audioSineTesterTrack = true;
+ track.__audioSineTesterContext = context;
+ track.__audioSineTesterOscillator = oscillator;
+ track.__audioSineTesterGain = gain;
+ return track;
+ }
+
+ function stopTrack(track) {
+ if (!track) return;
+ try { track.__audioSineTesterOscillator?.stop(); } catch {}
+ try { track.stop(); } catch {}
+ try { track.__audioSineTesterContext?.close(); } catch {}
+ }
+
+ async function startSine(candidateId, frequency) {
+ const candidate = senderCandidates.get(candidateId);
+ if (!candidate?.sender) return;
+
+ await stopSine(false);
+
+ const originalTrack = candidate.sender.track || candidate.originalTrack || null;
+ const sineTrack = makeSineTrack(frequency);
+ await candidate.sender.replaceTrack(sineTrack);
+
+ activeTest = {
+ candidateId,
+ sender: candidate.sender,
+ originalTrack,
+ sineTrack,
+ frequency,
+ startedAt: Date.now(),
+ };
+
+ candidate.originalTrack = originalTrack;
+ candidate.track = sineTrack;
+ log(`START ${frequency}Hz on ${candidateId}; original=${originalTrack?.id || 'null'}`);
+ renderPanel();
+ }
+
+ async function stopSine(shouldLog = true) {
+ if (!activeTest) return;
+ const test = activeTest;
+ activeTest = null;
+
+ try {
+ await test.sender.replaceTrack(test.originalTrack || null);
+ if (shouldLog) log(`STOP ${test.frequency}Hz on ${test.candidateId}; restored=${test.originalTrack?.id || 'null'}`);
+ } catch (error) {
+ log(`restore failed on ${test.candidateId}: ${error.message || error}`);
+ } finally {
+ stopTrack(test.sineTrack);
+ }
+ renderPanel();
+ }
+
+ function dump() {
+ return {
+ activeTest: activeTest ? {
+ candidateId: activeTest.candidateId,
+ frequency: activeTest.frequency,
+ sineTrackId: activeTest.sineTrack.id,
+ originalTrackId: activeTest.originalTrack?.id || null,
+ seconds: Math.round((Date.now() - activeTest.startedAt) / 1000),
+ } : null,
+ senders: Array.from(senderCandidates.values())
+ .sort((a, b) => b.score - a.score)
+ .map((candidate) => ({
+ id: candidate.id,
+ active: activeTest?.candidateId === candidate.id,
+ trackId: candidate.sender.track?.id || candidate.track?.id || null,
+ trackLabel: candidate.sender.track?.label || candidate.track?.label || null,
+ originalTrackId: candidate.originalTrack?.id || null,
+ trackEnabled: candidate.sender.track?.enabled ?? null,
+ trackMuted: candidate.sender.track?.muted ?? null,
+ trackReadyState: candidate.sender.track?.readyState || null,
+ bytesSent: candidate.bytesSent,
+ packetsSent: candidate.packetsSent,
+ bytesDelta: candidate.bytesDelta,
+ packetsDelta: candidate.packetsDelta,
+ score: Math.round(candidate.score),
+ sources: Array.from(candidate.sources),
+ })),
+ };
+ }
+
+ function renderPanel() {
+ if (!document.body) return;
+ let panel = document.getElementById('audio-sine-tester-panel');
+ if (!panel) {
+ panel = document.createElement('div');
+ panel.id = 'audio-sine-tester-panel';
+ panel.style.cssText = [
+ 'position:fixed',
+ 'top:20px',
+ 'right:20px',
+ 'width:500px',
+ 'max-height:76vh',
+ 'z-index:999999',
+ 'box-sizing:border-box',
+ 'padding:12px',
+ 'border-radius:8px',
+ 'background:rgba(3,7,18,.96)',
+ 'color:#e5e7eb',
+ 'font:12px ui-monospace,Menlo,monospace',
+ 'box-shadow:0 12px 32px rgba(0,0,0,.35)',
+ 'overflow:auto',
+ ].join(';');
+ panel.innerHTML = `
+
+ Audio Sender Sine Tester
+
+
+
+
+
+
+
+
+ `;
+ document.body.appendChild(panel);
+
+ document.getElementById('audio-sine-copy').addEventListener('click', async () => {
+ const text = JSON.stringify(window.__audioSineTesterDump(), null, 2);
+ await navigator.clipboard?.writeText(text).catch(() => {});
+ console.log('[Audio Sine Tester dump]', text);
+ log('dump copied to clipboard and console');
+ });
+ document.getElementById('audio-sine-stop').addEventListener('click', () => stopSine());
+ }
+
+ const data = dump();
+ document.getElementById('audio-sine-summary').textContent =
+ `senders=${data.senders.length}, active=${data.activeTest ? `${data.activeTest.candidateId} ${data.activeTest.frequency}Hz ${data.activeTest.seconds}s` : 'none'}, time=${nowText()}`;
+
+ document.getElementById('audio-sine-list').innerHTML = data.senders.map((row) => {
+ const pct = Math.min(100, Math.max(row.bytesDelta / 20, row.packetsDelta * 2));
+ const active = row.active;
+ return `
+
+
+ ${active ? '*' : '-'} ${row.id}
+ score=${row.score}
+
+
+
+ track=${shortId(row.trackId)} label=${row.trackLabel || 'null'} muted=${row.trackMuted} bytesDelta=${row.bytesDelta} packetsDelta=${row.packetsDelta}
+
+
+
+
+
+
${row.sources.join(', ')}
+
- AI 原生选轨
- 未连接
-
-
- 音频输入：布局优先自动选轨
- 视频输入：远端视频轨
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- 等待交互...
-
- `;
-
- document.body.appendChild(panel);
-
- const toggle = document.getElementById('native-copilot-toggle');
- const recordButton = document.getElementById('native-copilot-record');
- const videoRecordButton = document.getElementById('native-copilot-record-video');
- const urlInput = document.getElementById('native-copilot-url');
- const assistantInput = document.getElementById('native-copilot-assistant');
- const sendAudioInput = document.getElementById('native-copilot-send-audio');
- const sendVideoInput = document.getElementById('native-copilot-send-video');
-
- sendAudioInput.checked = shouldSendAudioToAi();
- sendVideoInput.checked = shouldSendVideoToAi();
- sendAudioInput.addEventListener('change', () => {
- writeBoolSetting(AUDIO_UPLINK_STORAGE_KEY, sendAudioInput.checked);
- log(`AI audio uplink setting: ${sendAudioInput.checked ? 'on' : 'off'}`);
- });
- sendVideoInput.addEventListener('change', () => {
- writeBoolSetting(VIDEO_UPLINK_STORAGE_KEY, sendVideoInput.checked);
- log(`AI video uplink setting: ${sendVideoInput.checked ? 'on' : 'off'}`);
- });
- recordButton.addEventListener('click', toggleInputRecording);
- videoRecordButton.addEventListener('click', toggleVideoRecording);
- updateRecordButton();
- updateVideoRecordButton();
- updateSelectedInputLabel();
- updateModeButton();
-
- toggle.addEventListener('click', async () => {
- if (!isAiTakeover && !assistantInput.value.trim()) {
- alert('请先填写助手 ID。');
- return;
- }
-
- if (!isAiTakeover && !ensureUserTracksBeforeConnect()) {
- updateModeButton();
- return;
- }
-
- isAiTakeover = !isAiTakeover;
- updateModeButton();
- if (isAiTakeover) {
- const connected = await connectAi(urlInput.value.trim(), assistantInput.value.trim());
- if (connected) await attachAiOutputToBusiness();
- else {
- isAiTakeover = false;
- updateModeButton();
- }
- } else {
- await restoreBusinessAudio();
- disconnectAi();
- }
- });
- }
-
- function renderPanelSoon() {
- try { renderPanel(); } catch (error) { console.warn('[AI Native Copilot] panel error', error); }
- }
-
- if (document.readyState === 'loading') {
- document.addEventListener('DOMContentLoaded', renderPanelSoon, { once: true });
- } else {
- renderPanelSoon();
- }
- requestAnimationFrame(renderVu);
- setInterval(refreshVuFromStats, 250);
- setTimeout(renderPanelSoon, 1500);
- setTimeout(renderPanelSoon, 3000);
-})();
diff --git a/scripts/copilot-native.js b/scripts/copilot-native.js
deleted file mode 100644
index c340435..0000000
--- a/scripts/copilot-native.js
+++ /dev/null
@@ -1,1542 +0,0 @@
-// ==UserScript==
-// @name 12345 AI Native Track Copilot
-// @namespace http://tampermonkey.net/
-// @version 1.0
-// @description Native-track only bridge: no WebAudio washing, no captureStream, no MediaStreamDestination.
-// @author You
-// @match *://yhy-yw.22sj.cn:*/*
-// @grant none
-// @run-at document-start
-// ==/UserScript==
-
-(function () {
- 'use strict';
-
- const DEFAULT_SIGNALING_URL = 'ws://182.92.86.220:8000/ws/voice';
- const AUDIO_UPLINK_STORAGE_KEY = 'native-copilot-send-audio';
- const VIDEO_UPLINK_STORAGE_KEY = 'native-copilot-send-video';
-
- const originalGetUserMedia = navigator.mediaDevices?.getUserMedia?.bind(navigator.mediaDevices);
- const originalAddEventListener = RTCPeerConnection.prototype.addEventListener;
- const originalAddTrack = RTCPeerConnection.prototype.addTrack;
- const originalAddTransceiver = RTCPeerConnection.prototype.addTransceiver;
- const originalSetRemoteDescription = RTCPeerConnection.prototype.setRemoteDescription;
- const mediaSrcObjectDescriptor = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, 'srcObject');
-
- const remoteStreams = new Set();
- const ignoredTrackIds = new Set();
- const audioCandidates = new Map();
- const videoCandidates = new Map();
- const businessSenderCandidates = new Map();
- const localCaptureTrackIds = new Set();
- window.__aiNativeCopilotReceivers = window.__aiNativeCopilotReceivers || new Set();
-
- let aiPc = null;
- let aiWs = null;
- let aiDc = null;
- let aiAudioSender = null;
- let aiVideoSender = null;
-
- let businessAudioSender = null;
- let businessOriginalAudioTrack = null;
-
- let remoteUserAudioTrack = null;
- let remoteUserVideoTrack = null;
- let aiResponseAudioTrack = null;
- let remoteUserAudioLocked = false;
- let remoteUserVideoLocked = false;
-
- let isAiTakeover = false;
- let aiTrackAttachedToBusiness = false;
- let inputVuLevel = 0;
- let aiOutVuLevel = 0;
- let aiInputHealthTimer = null;
- let aiOutputHealthTimer = null;
- let assistantLine = null;
- let assistantBuffer = '';
- let inputRecorder = null;
- let inputRecordingChunks = [];
- let inputRecordingTrack = null;
- let inputRecordingStartedAt = 0;
- let videoRecorder = null;
- let videoRecordingChunks = [];
- let videoRecordingTrack = null;
- let videoRecordingStartedAt = 0;
-
- const receiverStatsMemory = new Map();
- const senderStatsMemory = new Map();
- const inboundStatsMemory = new Map();
- const outboundStatsMemory = new Map();
- const businessOutboundStats = {
- bytesSent: 0,
- packetsSent: 0,
- bytesDelta: 0,
- packetsDelta: 0,
- };
-
- function log(message) {
- console.log(`[AI Native Copilot] ${message}`);
- }
-
- function readBoolSetting(key, fallback) {
- try {
- const value = localStorage.getItem(key);
- if (value === null) return fallback;
- return value === 'true';
- } catch {
- return fallback;
- }
- }
-
- function writeBoolSetting(key, value) {
- try {
- localStorage.setItem(key, value ? 'true' : 'false');
- } catch {
- // Keep the live checkbox state even if storage is unavailable.
- }
- }
-
- function shouldSendAudioToAi() {
- return readBoolSetting(AUDIO_UPLINK_STORAGE_KEY, true);
- }
-
- function shouldSendVideoToAi() {
- return readBoolSetting(VIDEO_UPLINK_STORAGE_KEY, true);
- }
-
- function appendChatLine(role, text, important = false) {
- const box = document.getElementById('native-copilot-log');
- if (!box) return;
- if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = '';
-
- const row = document.createElement('div');
- row.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] ${role}: ${text}`;
- row.style.marginBottom = '4px';
- if (important) row.style.color = '#fcd34d';
- box.appendChild(row);
- while (box.children.length > 120) box.removeChild(box.firstChild);
- box.scrollTop = box.scrollHeight;
- return row;
- }
-
- function startAssistantLine() {
- assistantBuffer = '';
- assistantLine = null;
- }
-
- function appendAssistantDelta(delta) {
- if (!delta) return;
- const box = document.getElementById('native-copilot-log');
- if (!box) {
- console.log(`[AI Native Copilot] AI: ${delta}`);
- return;
- }
- if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = '';
- if (!assistantLine) assistantLine = appendChatLine('AI', '');
- assistantBuffer += delta;
- assistantLine.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] AI: ${assistantBuffer}`;
- box.scrollTop = box.scrollHeight;
- }
-
- function finishAssistantLine() {
- assistantLine = null;
- assistantBuffer = '';
- }
-
- function logUserTranscript(text) {
- if (!text?.trim()) return;
- finishAssistantLine();
- appendChatLine('用户', text.trim(), true);
- }
-
- function recorderMimeType() {
- if (window.MediaRecorder?.isTypeSupported?.('audio/webm;codecs=opus')) return 'audio/webm;codecs=opus';
- if (window.MediaRecorder?.isTypeSupported?.('audio/webm')) return 'audio/webm';
- return '';
- }
-
- function recordingFileName(track) {
- const stamp = new Date().toISOString().replace(/[:.]/g, '-');
- const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track';
- return `native-ai-input-${trackPart}-${stamp}.webm`;
- }
-
- function videoRecordingFileName(track) {
- const stamp = new Date().toISOString().replace(/[:.]/g, '-');
- const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track';
- return `native-ai-video-${trackPart}-${stamp}.webm`;
- }
-
- function updateRecordButton() {
- const button = document.getElementById('native-copilot-record');
- if (!button) return;
- const recording = inputRecorder?.state === 'recording';
- button.textContent = recording ? '停止录制 AI 输入' : '录制 AI 输入';
- button.style.background = recording ? '#dc2626' : '#2563eb';
- }
-
- function videoRecorderMimeType() {
- if (window.MediaRecorder?.isTypeSupported?.('video/webm;codecs=vp8')) return 'video/webm;codecs=vp8';
- if (window.MediaRecorder?.isTypeSupported?.('video/webm')) return 'video/webm';
- return '';
- }
-
- function updateVideoRecordButton() {
- const button = document.getElementById('native-copilot-record-video');
- if (!button) return;
- const recording = videoRecorder?.state === 'recording';
- button.textContent = recording ? '停止录制 AI 视频' : '录制 AI 视频';
- button.style.background = recording ? '#dc2626' : '#475569';
- }
-
- async function startInputRecording() {
- if (inputRecorder?.state === 'recording') return;
- chooseRemoteAudioTrack('record');
- if (!isUsableTrack(remoteUserAudioTrack)) {
- appendChatLine('录音', '没有可录制的 AI 输入音轨', true);
- return;
- }
- if (!window.MediaRecorder) {
- appendChatLine('录音', '当前浏览器不支持 MediaRecorder', true);
- return;
- }
-
- inputRecordingChunks = [];
- inputRecordingTrack = remoteUserAudioTrack;
- inputRecordingStartedAt = Date.now();
- const stream = new MediaStream([inputRecordingTrack]);
- const mimeType = recorderMimeType();
- inputRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined);
- inputRecorder.ondataavailable = (event) => {
- if (event.data?.size > 0) inputRecordingChunks.push(event.data);
- };
- inputRecorder.onstop = () => {
- const blob = new Blob(inputRecordingChunks, { type: inputRecorder?.mimeType || 'audio/webm' });
- const url = URL.createObjectURL(blob);
- const link = document.createElement('a');
- link.href = url;
- link.download = recordingFileName(inputRecordingTrack);
- document.body.appendChild(link);
- link.click();
- link.remove();
- setTimeout(() => URL.revokeObjectURL(url), 30000);
-
- const seconds = ((Date.now() - inputRecordingStartedAt) / 1000).toFixed(1);
- appendChatLine('录音', `已保存 ${link.download}（${seconds}s，${(blob.size / 1024).toFixed(1)} KB）`, true);
- inputRecorder = null;
- inputRecordingChunks = [];
- inputRecordingTrack = null;
- updateRecordButton();
- };
- inputRecorder.start(250);
- appendChatLine('录音', `开始录制 AI 输入：${describeTrack(inputRecordingTrack)}`, true);
- updateRecordButton();
- }
-
- function stopInputRecording() {
- if (inputRecorder?.state === 'recording') inputRecorder.stop();
- }
-
- function toggleInputRecording() {
- if (inputRecorder?.state === 'recording') stopInputRecording();
- else startInputRecording();
- }
-
- async function startVideoRecording() {
- if (videoRecorder?.state === 'recording') return;
- chooseRemoteVideoTrack('record-video');
- if (!remoteUserVideoLocked || !isUsableTrack(remoteUserVideoTrack)) {
- appendChatLine('录制视频', '没有可录制的上半部用户视频轨', true);
- return;
- }
- if (!window.MediaRecorder) {
- appendChatLine('录制视频', '当前浏览器不支持 MediaRecorder', true);
- return;
- }
-
- videoRecordingChunks = [];
- videoRecordingTrack = remoteUserVideoTrack;
- videoRecordingStartedAt = Date.now();
- const stream = new MediaStream([videoRecordingTrack]);
- const mimeType = videoRecorderMimeType();
- videoRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined);
- videoRecorder.ondataavailable = (event) => {
- if (event.data?.size > 0) videoRecordingChunks.push(event.data);
- };
- videoRecorder.onstop = () => {
- const blob = new Blob(videoRecordingChunks, { type: videoRecorder?.mimeType || 'video/webm' });
- const url = URL.createObjectURL(blob);
- const link = document.createElement('a');
- link.href = url;
- link.download = videoRecordingFileName(videoRecordingTrack);
- document.body.appendChild(link);
- link.click();
- link.remove();
- setTimeout(() => URL.revokeObjectURL(url), 30000);
-
- const seconds = ((Date.now() - videoRecordingStartedAt) / 1000).toFixed(1);
- appendChatLine('录制视频', `已保存 ${link.download}（${seconds}s，${(blob.size / 1024).toFixed(1)} KB）`, true);
- videoRecorder = null;
- videoRecordingChunks = [];
- videoRecordingTrack = null;
- updateVideoRecordButton();
- };
- videoRecorder.start(250);
- appendChatLine('录制视频', `开始录制 AI 视频：${describeTrack(videoRecordingTrack)}`, true);
- updateVideoRecordButton();
- }
-
- function stopVideoRecording() {
- if (videoRecorder?.state === 'recording') videoRecorder.stop();
- }
-
- function toggleVideoRecording() {
- if (videoRecorder?.state === 'recording') stopVideoRecording();
- else startVideoRecording();
- }
-
- function generatePcId() {
- const bytes = new Uint8Array(16);
- crypto.getRandomValues(bytes);
- return 'PC-' + Array.from(bytes).map((b) => b.toString(16).padStart(2, '0')).join('');
- }
-
- function isInternalPc(pc) {
- return Boolean(pc?.__aiNativeCopilotInternal);
- }
-
- function isUsableTrack(track) {
- return track && track.readyState === 'live' && !ignoredTrackIds.has(track.id);
- }
-
- function isLiveTrack(track) {
- return track && track.readyState === 'live';
- }
-
- function candidateFor(track, candidates) {
- let candidate = candidates.get(track.id);
- if (!candidate) {
- candidate = {
- track,
- receiver: null,
- reasons: new Set(),
- elements: new Set(),
- firstSeenAt: Date.now(),
- lastSeenAt: 0,
- lastPacketAt: 0,
- bytesReceived: 0,
- packetsReceived: 0,
- bytesDelta: 0,
- packetsDelta: 0,
- level: 0,
- score: 0,
- logged: false,
- };
- candidates.set(track.id, candidate);
- }
- candidate.track = track;
- candidate.lastSeenAt = Date.now();
- return candidate;
- }
-
- function businessSenderCandidateFor(sender, track, reason) {
- if (!sender) return null;
- const senderTrack = track || sender.track;
- if (senderTrack && senderTrack.kind !== 'audio') return null;
-
- const id = sender.__aiNativeCopilotBusinessSenderId || `business-sender-${businessSenderCandidates.size + 1}`;
- sender.__aiNativeCopilotBusinessSenderId = id;
-
- let candidate = businessSenderCandidates.get(id);
- if (!candidate) {
- candidate = {
- id,
- sender,
- track: senderTrack || null,
- originalTrack: senderTrack || null,
- sources: new Set(),
- bytesSent: 0,
- packetsSent: 0,
- bytesDelta: 0,
- packetsDelta: 0,
- score: 0,
- logged: false,
- };
- businessSenderCandidates.set(id, candidate);
- }
-
- candidate.sender = sender;
- candidate.track = senderTrack || sender.track || candidate.track;
- if (!isAiTakeover && senderTrack && senderTrack !== aiResponseAudioTrack) {
- candidate.originalTrack = senderTrack;
- }
- candidate.sources.add(reason);
- wrapBusinessSender(sender);
- return candidate;
- }
-
- function findReceiverForTrack(track) {
- if (!track) return null;
- return Array.from(window.__aiNativeCopilotReceivers || [])
- .find((receiver) => receiver.track === track) || null;
- }
-
- function rememberReceiver(receiver, reason) {
- if (!receiver?.track || isInternalPc(receiver.transport)) return;
- window.__aiNativeCopilotReceivers.add(receiver);
- const track = receiver.track;
- if (!isUsableTrack(track)) return;
-
- if (track.kind === 'audio') {
- const candidate = candidateFor(track, audioCandidates);
- candidate.receiver = receiver;
- candidate.reasons.add(reason);
- } else if (track.kind === 'video') {
- const candidate = candidateFor(track, videoCandidates);
- candidate.receiver = receiver;
- candidate.reasons.add(reason);
- chooseRemoteVideoTrack(reason);
- }
- }
-
- function setVuWidth(id, level) {
- const el = document.getElementById(id);
- if (!el) return;
- el.style.width = `${Math.min(100, level * 2500)}%`;
- }
-
- function renderVu() {
- inputVuLevel *= 0.88;
- aiOutVuLevel *= 0.88;
- setVuWidth('native-copilot-input-vu', inputVuLevel);
- setVuWidth('native-copilot-ai-vu', aiOutVuLevel);
- requestAnimationFrame(renderVu);
- }
-
- function levelFromStats(stat, memory) {
- let level = typeof stat.audioLevel === 'number' ? stat.audioLevel : 0;
- if (typeof stat.totalAudioEnergy === 'number' && typeof stat.totalSamplesDuration === 'number') {
- const prev = memory.get(stat.id);
- memory.set(stat.id, {
- energy: stat.totalAudioEnergy,
- duration: stat.totalSamplesDuration,
- });
- if (prev) {
- const energyDelta = stat.totalAudioEnergy - prev.energy;
- const durationDelta = stat.totalSamplesDuration - prev.duration;
- if (energyDelta > 0 && durationDelta > 0) {
- level = Math.max(level, Math.sqrt(energyDelta / durationDelta));
- }
- }
- }
- return level;
- }
-
- async function refreshVuFromStats() {
- try {
- if (!(await checkLockedUserTracks('stats-before'))) return;
- await refreshAudioCandidateStats();
- chooseRemoteAudioTrack('stats');
-
- await refreshBusinessSenderStats();
- chooseBusinessAudioSender('stats');
- await checkLockedUserTracks('stats-after');
- } catch (error) {
- // Stats APIs are best-effort across browsers.
- }
- }
-
- async function refreshBusinessSenderStats() {
- for (const candidate of businessSenderCandidates.values()) {
- const sender = candidate.sender;
- if (!sender?.getStats) continue;
- candidate.track = sender.track || candidate.track;
-
- const report = await sender.getStats();
- report.forEach((stat) => {
- const kind = stat.kind || stat.mediaType;
- if (stat.type !== 'outbound-rtp' || kind !== 'audio') return;
-
- const key = `${candidate.id}:${stat.id}`;
- const prev = outboundStatsMemory.get(key);
- const bytes = typeof stat.bytesSent === 'number' ? stat.bytesSent : 0;
- const packets = typeof stat.packetsSent === 'number' ? stat.packetsSent : 0;
- candidate.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0;
- candidate.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0;
- candidate.bytesSent = bytes;
- candidate.packetsSent = packets;
- outboundStatsMemory.set(key, { bytes, packets });
-
- if (candidate.sender === businessAudioSender) {
- businessOutboundStats.bytesDelta = candidate.bytesDelta;
- businessOutboundStats.packetsDelta = candidate.packetsDelta;
- businessOutboundStats.bytesSent = candidate.bytesSent;
- businessOutboundStats.packetsSent = candidate.packetsSent;
- aiOutVuLevel = Math.max(
- aiOutVuLevel,
- levelFromStats(stat, senderStatsMemory),
- outputVuLevelFromStats()
- );
- }
- });
- }
- }
-
- async function refreshAudioCandidateStats() {
- for (const candidate of audioCandidates.values()) {
- if (!isUsableTrack(candidate.track)) continue;
- const receiver = candidate.receiver || findReceiverForTrack(candidate.track);
- if (!receiver?.getStats) continue;
- candidate.receiver = receiver;
-
- const report = await receiver.getStats();
- report.forEach((stat) => {
- const kind = stat.kind || stat.mediaType;
- if (stat.type !== 'inbound-rtp' || kind !== 'audio') return;
-
- const prev = inboundStatsMemory.get(stat.id);
- const bytes = typeof stat.bytesReceived === 'number' ? stat.bytesReceived : 0;
- const packets = typeof stat.packetsReceived === 'number' ? stat.packetsReceived : 0;
- const bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0;
- const packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0;
- inboundStatsMemory.set(stat.id, { bytes, packets });
-
- candidate.bytesReceived = bytes;
- candidate.packetsReceived = packets;
- candidate.bytesDelta = bytesDelta;
- candidate.packetsDelta = packetsDelta;
- candidate.level = levelFromStats(stat, receiverStatsMemory);
- if (bytesDelta > 0 || packetsDelta > 0 || candidate.level > 0) {
- candidate.lastPacketAt = Date.now();
- }
-
- if (candidate.track === remoteUserAudioTrack) {
- inputVuLevel = Math.max(inputVuLevel, inputVuLevelFromCandidate(candidate));
- }
- });
- }
- }
-
- async function outboundAudioBytes(sender) {
- if (!sender?.getStats) return null;
- try {
- let total = 0;
- const report = await sender.getStats();
- report.forEach((stat) => {
- const kind = stat.kind || stat.mediaType;
- if (stat.type === 'outbound-rtp' && kind === 'audio' && typeof stat.bytesSent === 'number') {
- total += stat.bytesSent;
- }
- });
- return total;
- } catch {
- return null;
- }
- }
-
- function aiReadyForMedia() {
- if (!aiPc || !aiPc.remoteDescription) return false;
- return (
- aiPc.connectionState === 'connected'
- || aiPc.iceConnectionState === 'connected'
- || aiPc.iceConnectionState === 'completed'
- );
- }
-
- function aiStateLabel() {
- if (!aiPc) return 'pc=null';
- return `conn=${aiPc.connectionState},ice=${aiPc.iceConnectionState},signaling=${aiPc.signalingState},remote=${aiPc.remoteDescription?.type || 'none'}`;
- }
-
- function waitForAiMediaReady(timeoutMs = 10000) {
- if (aiReadyForMedia()) return Promise.resolve(true);
- return new Promise((resolve) => {
- const startedAt = Date.now();
- const timer = setInterval(() => {
- if (aiReadyForMedia()) {
- clearInterval(timer);
- resolve(true);
- } else if (Date.now() - startedAt >= timeoutMs) {
- clearInterval(timer);
- resolve(false);
- }
- }, 250);
- });
- }
-
- function scheduleAiInputHealthCheck(track) {
- if (aiInputHealthTimer) clearTimeout(aiInputHealthTimer);
- aiInputHealthTimer = setTimeout(async () => {
- if (!aiAudioSender || aiAudioSender.track !== track) return;
- log(`checking native AI input after media connects: ${aiStateLabel()}`);
- const ready = await waitForAiMediaReady(10000);
- if (!ready || !aiAudioSender || aiAudioSender.track !== track) {
- log(`native AI input could not be verified: ${aiStateLabel()}`);
- return;
- }
-
- const before = await outboundAudioBytes(aiAudioSender);
- await new Promise((resolve) => setTimeout(resolve, 2500));
- if (!aiAudioSender || aiAudioSender.track !== track) return;
- const after = await outboundAudioBytes(aiAudioSender);
- log(`native AI input packet check: before=${before}, after=${after}, ${aiStateLabel()}`, true);
- }, 500);
- }
-
- function describeTrack(track) {
- if (!track) return 'none';
- return `${track.id}${track.muted ? ' muted' : ''}`;
- }
-
- function elementLayoutInfo(element) {
- if (!element?.getBoundingClientRect || !document.documentElement.contains(element)) return null;
- const rect = element.getBoundingClientRect();
- const viewportHeight = window.innerHeight || document.documentElement.clientHeight || 1;
- const centerY = rect.top + rect.height / 2;
- const visibleWidth = Math.max(0, Math.min(rect.right, window.innerWidth || document.documentElement.clientWidth || rect.right) - Math.max(rect.left, 0));
- const visibleHeight = Math.max(0, Math.min(rect.bottom, viewportHeight) - Math.max(rect.top, 0));
- return {
- tag: element.tagName,
- top: Math.round(rect.top),
- left: Math.round(rect.left),
- width: Math.round(rect.width),
- height: Math.round(rect.height),
- centerY: Math.round(centerY),
- upperHalf: centerY <= viewportHeight * 0.55,
- visibleArea: Math.round(visibleWidth * visibleHeight),
- paused: Boolean(element.paused),
- muted: Boolean(element.muted),
- volume: typeof element.volume === 'number' ? element.volume : null,
- };
- }
-
- function candidateLayoutScore(candidate) {
- let score = 0;
- for (const element of candidate?.elements || []) {
- const info = elementLayoutInfo(element);
- if (!info) continue;
- if (info.upperHalf) score += 9000;
- else score -= 2500;
- if (info.visibleArea > 0) score += Math.min(2500, info.visibleArea / 80);
- if (info.tag === 'VIDEO') score += 1200;
- if (info.tag === 'AUDIO') score += 300;
- if (!info.paused) score += 700;
- if (!info.muted && (info.volume ?? 1) > 0) score += 500;
- }
- return score;
- }
-
- function hasUpperHalfElement(candidate) {
- return Array.from(candidate?.elements || []).some((element) => elementLayoutInfo(element)?.upperHalf);
- }
-
- function lockedTrackUsable(track, candidates) {
- const candidate = candidates.get(track?.id);
- return Boolean(candidate && isUsableTrack(candidate.track) && !candidate.track.muted);
- }
-
- async function handleLockedUserTrackLost(kind, reason) {
- const message = `${kind === 'audio' ? '远端音频' : '远端视频'}轨道失效（${reason}），已断开 AI`;
- appendChatLine('选轨', message, true);
- log(message);
- if (kind === 'audio') {
- remoteUserAudioLocked = false;
- remoteUserAudioTrack = null;
- } else {
- remoteUserVideoLocked = false;
- remoteUserVideoTrack = null;
- }
- if (isAiTakeover) {
- isAiTakeover = false;
- updateModeButton();
- await restoreBusinessAudio();
- disconnectAi();
- }
- }
-
- async function checkLockedUserTracks(reason) {
- if (remoteUserAudioLocked && !lockedTrackUsable(remoteUserAudioTrack, audioCandidates)) {
- await handleLockedUserTrackLost('audio', reason);
- return false;
- }
- if (remoteUserVideoLocked && !lockedTrackUsable(remoteUserVideoTrack, videoCandidates)) {
- await handleLockedUserTrackLost('video', reason);
- return false;
- }
- return true;
- }
-
- function scoreAudioCandidate(candidate) {
- if (!candidate || !isUsableTrack(candidate.track)) return -1;
- const hasRecentPackets = Date.now() - candidate.lastPacketAt < 3000;
- const hasAudibleLevel = candidate.level >= 0.0015;
- let score = 0;
- if (candidate.track.muted) score -= 5000;
- else score += 800;
- score += candidateLayoutScore(candidate);
- if (candidate.receiver) score += 600;
- if (candidate.bytesReceived > 0) score += 100;
- if (candidate.bytesDelta > 0) score += 500 + Math.min(900, candidate.bytesDelta * 2);
- if (candidate.packetsDelta > 0) score += 300 + Math.min(500, candidate.packetsDelta * 8);
- if (hasRecentPackets) score += 400;
- if (hasAudibleLevel) score += 5000;
- score += Math.min(12000, candidate.level * 900000);
- if (!hasAudibleLevel && hasRecentPackets) score -= 1800;
- if (candidate.reasons.has('media-srcObject') || candidate.reasons.has('media-scan')) score += 80;
- return score;
- }
-
- function scoreVideoCandidate(candidate) {
- if (!candidate || !isUsableTrack(candidate.track)) return -1;
- let score = 0;
- if (candidate.track.muted) score -= 1000;
- else score += 500;
- if (candidate.receiver) score += 600;
- score += candidateLayoutScore(candidate);
- for (const element of candidate.elements || []) {
- const info = elementLayoutInfo(element);
- if (!info) continue;
- if (info.tag === 'VIDEO') score += 2500;
- if (info.visibleArea > 0) score += Math.min(4000, info.visibleArea / 50);
- }
- if (candidate.reasons.has('media-srcObject') || candidate.reasons.has('media-scan')) score += 100;
- return score;
- }
-
- function inputVuLevelFromCandidate(candidate) {
- if (!candidate) return 0;
- return Math.max(
- candidate.level,
- Math.min(0.04, candidate.bytesDelta / 16000),
- Math.min(0.02, candidate.packetsDelta / 800)
- );
- }
-
- function updateSelectedInputLabel() {
- const label = document.getElementById('native-copilot-selected-input');
- if (!label) return;
- const candidate = audioCandidates.get(remoteUserAudioTrack?.id);
- if (!candidate) {
- label.textContent = '等待远端音轨';
- return;
- }
- const layout = Array.from(candidate.elements || []).map(elementLayoutInfo).find(Boolean);
- const where = layout ? (layout.upperHalf ? 'upper' : 'lower') : 'no-layout';
- label.textContent = `${remoteUserAudioTrack.id.slice(0, 8)} ${where} level=${candidate.level.toFixed(4)} score=${Math.round(candidate.score || scoreAudioCandidate(candidate))}`;
- }
-
- function outputVuLevelFromStats() {
- return Math.max(
- Math.min(0.12, businessOutboundStats.bytesDelta / 12000),
- Math.min(0.04, businessOutboundStats.packetsDelta / 500)
- );
- }
-
- function isKnownRemoteAudioTrack(track) {
- return Boolean(track && (
- track === remoteUserAudioTrack
- || audioCandidates.has(track.id)
- ));
- }
-
- function isAiOutputTrack(track) {
- return Boolean(track && track === aiResponseAudioTrack);
- }
-
- function scoreBusinessSenderCandidate(candidate) {
- if (!candidate?.sender) return -1;
- const track = candidate.sender.track || candidate.track;
- let score = 0;
-
- if (track?.kind === 'audio') score += 500;
- else if (!track) score -= 1000;
-
- if (track?.readyState === 'live') score += 700;
- if (track && !track.muted) score += 300;
- if (candidate.sources.has('pc-addTrack')) score += 5000;
- if (candidate.sources.has('pc-addTransceiver')) score -= 1000;
- if (track && localCaptureTrackIds.has(track.id)) score += 2500;
- if (candidate.originalTrack && localCaptureTrackIds.has(candidate.originalTrack.id)) score += 2500;
- if (isKnownRemoteAudioTrack(track)) score -= 9000;
- if (candidate.originalTrack && isKnownRemoteAudioTrack(candidate.originalTrack)) score -= 9000;
- if (isAiOutputTrack(track)) score += 4500;
- if (candidate.bytesSent > 0) score += 600;
- if (candidate.bytesDelta > 0) score += 1600 + Math.min(1800, candidate.bytesDelta * 4);
- if (candidate.packetsDelta > 0) score += 800 + Math.min(900, candidate.packetsDelta * 10);
-
- return score;
- }
-
- function chooseBusinessAudioSender(reason) {
- let best = null;
- let bestScore = -1;
- for (const candidate of businessSenderCandidates.values()) {
- candidate.score = scoreBusinessSenderCandidate(candidate);
- if (candidate.score > bestScore) {
- best = candidate;
- bestScore = candidate.score;
- }
- }
-
- if (!best || bestScore < 0) return false;
- if (businessAudioSender === best.sender) return true;
-
- businessAudioSender = best.sender;
- businessOriginalAudioTrack = best.originalTrack || best.sender.track || businessOriginalAudioTrack;
- log(`selected business audio sender (${reason}): ${best.id}, track=${describeTrack(best.sender.track)}, score=${Math.round(bestScore)}, sources=${Array.from(best.sources).join('+')}`, true);
- attachAiOutputToBusiness();
- return true;
- }
-
- function chooseRemoteAudioTrack(reason) {
- const lockedAudio = audioCandidates.get(remoteUserAudioTrack?.id);
- if (remoteUserAudioLocked && lockedAudio && isUsableTrack(lockedAudio.track) && !lockedAudio.track.muted) {
- updateSelectedInputLabel();
- return true;
- }
- if (remoteUserAudioLocked) return false;
-
- let best = null;
- let bestScore = -1;
- for (const candidate of audioCandidates.values()) {
- candidate.score = scoreAudioCandidate(candidate);
- if (candidate.score > bestScore) {
- best = candidate;
- bestScore = candidate.score;
- }
- }
-
- if (!best || bestScore < 0) return false;
- if (remoteUserAudioTrack === best.track) {
- updateSelectedInputLabel();
- return true;
- }
-
- const current = audioCandidates.get(remoteUserAudioTrack?.id);
- const currentScore = scoreAudioCandidate(current);
- const currentHealthy = current && isUsableTrack(current.track) && !current.track.muted
- && (current.level >= 0.0015 || hasUpperHalfElement(current))
- && Date.now() - current.lastPacketAt < 3000;
- if (currentHealthy && bestScore < currentScore + 1800) {
- updateSelectedInputLabel();
- return true;
- }
-
- remoteUserAudioTrack = best.track;
- remoteUserAudioLocked = hasUpperHalfElement(best);
- log(`${remoteUserAudioLocked ? 'locked' : 'selected'} remote audio track (${reason}): ${describeTrack(best.track)}, score=${Math.round(bestScore)}, level=${best.level.toFixed(4)}, bytesDelta=${best.bytesDelta}, packetsDelta=${best.packetsDelta}`, true);
- updateSelectedInputLabel();
- attachRemoteInputToAi();
- return true;
- }
-
- function chooseRemoteVideoTrack(reason) {
- const lockedVideo = videoCandidates.get(remoteUserVideoTrack?.id);
- if (remoteUserVideoLocked && lockedVideo && isUsableTrack(lockedVideo.track) && !lockedVideo.track.muted) {
- return true;
- }
- if (remoteUserVideoLocked) return false;
-
- let best = null;
- let bestScore = -1;
- for (const candidate of videoCandidates.values()) {
- candidate.score = scoreVideoCandidate(candidate);
- if (candidate.score > bestScore) {
- best = candidate;
- bestScore = candidate.score;
- }
- }
-
- if (!best || bestScore < 0) return false;
- if (remoteUserVideoTrack === best.track) return true;
-
- const current = videoCandidates.get(remoteUserVideoTrack?.id);
- const currentScore = scoreVideoCandidate(current);
- if (hasUpperHalfElement(current) && bestScore < currentScore + 1800) return true;
-
- remoteUserVideoTrack = best.track;
- remoteUserVideoLocked = hasUpperHalfElement(best);
- log(`${remoteUserVideoLocked ? 'locked' : 'selected'} remote video track (${reason}): ${describeTrack(best.track)}, score=${Math.round(bestScore)}, layout=${Math.round(candidateLayoutScore(best))}`, true);
- attachRemoteVideoToAi();
- return true;
- }
-
- function ensureUserTracksBeforeConnect() {
- chooseRemoteAudioTrack('pre-connect');
- chooseRemoteVideoTrack('pre-connect');
-
- if (shouldSendAudioToAi() && (!remoteUserAudioLocked || !lockedTrackUsable(remoteUserAudioTrack, audioCandidates))) {
- appendChatLine('选轨', '未锁定上半部用户音频轨，暂不连接 AI', true);
- return false;
- }
- if (shouldSendVideoToAi() && (!remoteUserVideoLocked || !lockedTrackUsable(remoteUserVideoTrack, videoCandidates))) {
- appendChatLine('选轨', '未锁定上半部用户视频轨，暂不连接 AI', true);
- return false;
- }
- return true;
- }
-
- function rememberRemoteTrack(track, reason, element = null) {
- if (!isUsableTrack(track)) return;
-
- if (track.kind === 'audio') {
- const candidate = candidateFor(track, audioCandidates);
- candidate.receiver = candidate.receiver || findReceiverForTrack(track);
- candidate.reasons.add(reason);
- if (element) candidate.elements.add(element);
- if (!candidate.logged) {
- candidate.logged = true;
- log(`captured remote audio candidate (${reason}): ${describeTrack(track)}`, true);
- }
- chooseRemoteAudioTrack(reason);
- return;
- }
-
- if (track.kind === 'video') {
- const candidate = candidateFor(track, videoCandidates);
- candidate.receiver = candidate.receiver || findReceiverForTrack(track);
- candidate.reasons.add(reason);
- if (element) candidate.elements.add(element);
- if (!candidate.logged) {
- candidate.logged = true;
- log(`captured remote video candidate (${reason}): ${describeTrack(track)}`, true);
- }
- chooseRemoteVideoTrack(reason);
- }
- }
-
- function rememberRemoteStream(stream, reason, element = null) {
- if (!stream || !(stream instanceof MediaStream)) return;
- if (stream.__aiNativeCopilotInternal) return;
-
- const liveTracks = stream.getTracks().filter(isUsableTrack);
- if (liveTracks.length === 0) return;
-
- remoteStreams.add(stream);
- for (const track of liveTracks) rememberRemoteTrack(track, reason, element);
- chooseRemoteVideoTrack(reason);
- }
-
- function rememberBusinessAudioSender(sender, track, reason) {
- const candidate = businessSenderCandidateFor(sender, track, reason);
- if (!candidate) return;
- if (!candidate.logged) {
- candidate.logged = true;
- log(`captured business audio sender candidate (${reason}): ${candidate.id}, track=${describeTrack(candidate.track)}`, true);
- }
- chooseBusinessAudioSender(reason);
- }
-
- function wrapBusinessSender(sender) {
- if (!sender || sender.__aiNativeCopilotWrapped) return;
- const originalReplaceTrack = sender.replaceTrack?.bind(sender);
- if (!originalReplaceTrack) return;
-
- sender.replaceTrack = async function (track) {
- const result = await originalReplaceTrack(track);
- const candidate = businessSenderCandidateFor(sender, track, 'replaceTrack');
- if (candidate) {
- candidate.track = track || sender.track || candidate.track;
- }
- if (!isAiTakeover && track?.kind === 'audio' && track !== aiResponseAudioTrack) {
- if (candidate) candidate.originalTrack = track;
- chooseBusinessAudioSender('replaceTrack');
- }
- return result;
- };
- sender.__aiNativeCopilotWrapped = true;
- }
-
- async function attachRemoteInputToAi() {
- if (!shouldSendAudioToAi()) return false;
- if (!aiAudioSender || !isUsableTrack(remoteUserAudioTrack)) return false;
- if (aiAudioSender.track === remoteUserAudioTrack) return true;
-
- try {
- await aiAudioSender.replaceTrack(remoteUserAudioTrack);
- log(`AI input uses native remote audio track (${remoteUserAudioTrack.id})`, true);
- scheduleAiInputHealthCheck(remoteUserAudioTrack);
- return true;
- } catch (error) {
- log(`failed to attach native remote audio to AI: ${error.message || error}`);
- return false;
- }
- }
-
- async function attachRemoteVideoToAi() {
- if (!shouldSendVideoToAi()) return false;
- if (!aiVideoSender || !isUsableTrack(remoteUserVideoTrack)) return false;
- if (aiVideoSender.track === remoteUserVideoTrack) return true;
-
- try {
- await aiVideoSender.replaceTrack(remoteUserVideoTrack);
- log('AI video uses native remote video track', true);
- return true;
- } catch (error) {
- log(`failed to attach native remote video to AI: ${error.message || error}`);
- return false;
- }
- }
-
- async function attachAiOutputToBusiness() {
- chooseBusinessAudioSender('before-ai-output');
- if (!isAiTakeover || !businessAudioSender || !isLiveTrack(aiResponseAudioTrack)) {
- log(`AI output waiting: takeover=${isAiTakeover}, businessSender=${Boolean(businessAudioSender)}, aiTrack=${describeTrack(aiResponseAudioTrack)}`);
- return false;
- }
- if (businessAudioSender.track === aiResponseAudioTrack) return true;
-
- try {
- await businessAudioSender.replaceTrack(aiResponseAudioTrack);
- aiTrackAttachedToBusiness = true;
- log(`business call now uses native AI response audio track (${aiResponseAudioTrack.id})`, true);
- scheduleAiOutputHealthCheck(aiResponseAudioTrack);
- return true;
- } catch (error) {
- aiTrackAttachedToBusiness = false;
- log(`failed to attach AI audio to business call: ${error.message || error}`);
- return false;
- }
- }
-
- function scheduleAiOutputHealthCheck(track) {
- if (aiOutputHealthTimer) clearTimeout(aiOutputHealthTimer);
- aiOutputHealthTimer = setTimeout(async () => {
- if (!businessAudioSender || businessAudioSender.track !== track) {
- log(`AI output check skipped: senderTrack=${describeTrack(businessAudioSender?.track)}, aiTrack=${describeTrack(track)}`);
- return;
- }
-
- const before = await outboundAudioBytes(businessAudioSender);
- await new Promise((resolve) => setTimeout(resolve, 2500));
- if (!businessAudioSender || businessAudioSender.track !== track) return;
- const after = await outboundAudioBytes(businessAudioSender);
- log(`AI output packet check: before=${before}, after=${after}, senderTrack=${describeTrack(businessAudioSender.track)}`, true);
- }, 800);
- }
-
- async function restoreBusinessAudio() {
- if (!businessAudioSender || !aiTrackAttachedToBusiness) return;
-
- try {
- await businessAudioSender.replaceTrack(businessOriginalAudioTrack || null);
- log('business call restored original audio track', true);
- } catch (error) {
- log(`failed to restore business audio track: ${error.message || error}`);
- } finally {
- aiTrackAttachedToBusiness = false;
- }
- }
-
- function wrapTrackListener(listener) {
- if (!listener || listener.__aiNativeCopilotWrapped) return listener;
-
- const wrapped = function (event) {
- if (!isInternalPc(this) && event?.track) {
- rememberReceiver(event.receiver, 'pc-track');
- rememberRemoteTrack(event.track, 'pc-track');
- for (const stream of event.streams || []) rememberRemoteStream(stream, 'pc-track-stream');
- }
-
- if (typeof listener === 'function') return listener.apply(this, arguments);
- if (listener?.handleEvent) return listener.handleEvent(event);
- return undefined;
- };
-
- wrapped.__aiNativeCopilotWrapped = true;
- return wrapped;
- }
-
- RTCPeerConnection.prototype.addEventListener = function (type, listener, options) {
- if (type === 'track') {
- return originalAddEventListener.call(this, type, wrapTrackListener(listener), options);
- }
- return originalAddEventListener.call(this, type, listener, options);
- };
-
- try {
- const descriptor = Object.getOwnPropertyDescriptor(RTCPeerConnection.prototype, 'ontrack');
- Object.defineProperty(RTCPeerConnection.prototype, 'ontrack', {
- configurable: true,
- enumerable: true,
- get() {
- return descriptor?.get ? descriptor.get.call(this) : this.__aiNativeCopilotOnTrack || null;
- },
- set(listener) {
- const wrapped = wrapTrackListener(listener);
- this.__aiNativeCopilotOnTrack = wrapped;
- if (descriptor?.set) descriptor.set.call(this, wrapped);
- else this.addEventListener('track', wrapped);
- },
- });
- } catch (error) {
- console.warn('[AI Native Copilot] failed to wrap ontrack', error);
- }
-
- RTCPeerConnection.prototype.addTrack = function (track, ...streams) {
- const sender = originalAddTrack.call(this, track, ...streams);
- if (!isInternalPc(this) && track?.kind === 'audio') {
- rememberBusinessAudioSender(sender, track, 'pc-addTrack');
- }
- return sender;
- };
-
- if (originalAddTransceiver) {
- RTCPeerConnection.prototype.addTransceiver = function (trackOrKind, init) {
- const transceiver = originalAddTransceiver.call(this, trackOrKind, init);
- if (!isInternalPc(this)) {
- const kind = trackOrKind instanceof MediaStreamTrack ? trackOrKind.kind : trackOrKind;
- if (kind === 'audio') {
- const track = trackOrKind instanceof MediaStreamTrack ? trackOrKind : transceiver.sender?.track;
- rememberBusinessAudioSender(transceiver.sender, track, 'pc-addTransceiver');
- }
- }
- return transceiver;
- };
- }
-
- RTCPeerConnection.prototype.setRemoteDescription = function () {
- const result = originalSetRemoteDescription.apply(this, arguments);
- if (!isInternalPc(this)) {
- Promise.resolve(result).then(() => {
- try {
- for (const receiver of this.getReceivers?.() || []) {
- rememberReceiver(receiver, 'receiver-after-srd');
- rememberRemoteTrack(receiver.track, 'receiver-after-srd');
- }
- for (const sender of this.getSenders?.() || []) {
- rememberBusinessAudioSender(sender, sender.track, 'sender-after-srd');
- }
- for (const stream of this.getRemoteStreams?.() || []) {
- rememberRemoteStream(stream, 'remote-stream-after-srd');
- }
- } catch (error) {
- console.warn('[AI Native Copilot] failed to scan receivers', error);
- }
- });
- }
- return result;
- };
-
- if (mediaSrcObjectDescriptor?.set) {
- Object.defineProperty(HTMLMediaElement.prototype, 'srcObject', {
- configurable: true,
- enumerable: mediaSrcObjectDescriptor.enumerable,
- get() {
- return mediaSrcObjectDescriptor.get.call(this);
- },
- set(stream) {
- if ((this.tagName === 'AUDIO' || this.tagName === 'VIDEO') && !this.__aiNativeCopilotElement) {
- rememberRemoteStream(stream, 'media-srcObject', this);
- }
- return mediaSrcObjectDescriptor.set.call(this, stream);
- },
- });
- }
-
- if (originalGetUserMedia) {
- navigator.mediaDevices.getUserMedia = async function (constraints) {
- const stream = await originalGetUserMedia(constraints);
- for (const track of stream.getAudioTracks()) {
- localCaptureTrackIds.add(track.id);
- }
- chooseBusinessAudioSender('getUserMedia');
- renderPanelSoon();
- return stream;
- };
- }
-
- setInterval(async () => {
- if (!(await checkLockedUserTracks('scan'))) return;
- for (const element of document.querySelectorAll('audio, video')) {
- if (element.__aiNativeCopilotElement) continue;
- rememberRemoteStream(element.srcObject, 'media-scan', element);
- }
-
- for (const stream of Array.from(remoteStreams)) {
- rememberRemoteStream(stream, 'stream-rescan');
- }
- await checkLockedUserTracks('scan-after');
- }, 1000);
-
- window.__aiNativeCopilotDump = function () {
- const audio = Array.from(audioCandidates.values()).map((candidate) => ({
- id: candidate.track.id,
- label: candidate.track.label,
- selected: candidate.track === remoteUserAudioTrack,
- enabled: candidate.track.enabled,
- muted: candidate.track.muted,
- readyState: candidate.track.readyState,
- hasReceiver: Boolean(candidate.receiver || findReceiverForTrack(candidate.track)),
- bytesReceived: candidate.bytesReceived,
- packetsReceived: candidate.packetsReceived,
- bytesDelta: candidate.bytesDelta,
- packetsDelta: candidate.packetsDelta,
- level: candidate.level,
- audible: candidate.level >= 0.0015,
- layoutScore: Math.round(candidateLayoutScore(candidate)),
- elements: Array.from(candidate.elements || []).map(elementLayoutInfo).filter(Boolean),
- vuLevel: inputVuLevelFromCandidate(candidate),
- score: Math.round(scoreAudioCandidate(candidate)),
- reasons: Array.from(candidate.reasons),
- }));
- const video = Array.from(videoCandidates.values()).map((candidate) => ({
- id: candidate.track.id,
- label: candidate.track.label,
- selected: candidate.track === remoteUserVideoTrack,
- muted: candidate.track.muted,
- readyState: candidate.track.readyState,
- hasReceiver: Boolean(candidate.receiver || findReceiverForTrack(candidate.track)),
- layoutScore: Math.round(candidateLayoutScore(candidate)),
- score: Math.round(scoreVideoCandidate(candidate)),
- elements: Array.from(candidate.elements || []).map(elementLayoutInfo).filter(Boolean),
- reasons: Array.from(candidate.reasons),
- }));
- const businessSenders = Array.from(businessSenderCandidates.values()).map((candidate) => ({
- id: candidate.id,
- selected: candidate.sender === businessAudioSender,
- track: describeTrack(candidate.sender.track || candidate.track),
- originalTrack: describeTrack(candidate.originalTrack),
- isRemoteTrack: isKnownRemoteAudioTrack(candidate.sender.track || candidate.track),
- isLocalCaptureTrack: Boolean(candidate.sender.track && localCaptureTrackIds.has(candidate.sender.track.id)),
- bytesSent: candidate.bytesSent,
- packetsSent: candidate.packetsSent,
- bytesDelta: candidate.bytesDelta,
- packetsDelta: candidate.packetsDelta,
- score: Math.round(scoreBusinessSenderCandidate(candidate)),
- sources: Array.from(candidate.sources),
- }));
- return {
- selectedAudio: describeTrack(remoteUserAudioTrack),
- selectedVideo: describeTrack(remoteUserVideoTrack),
- audioLocked: remoteUserAudioLocked,
- videoLocked: remoteUserVideoLocked,
- aiInputTrack: describeTrack(aiAudioSender?.track),
- aiOutputTrack: describeTrack(aiResponseAudioTrack),
- businessSenderTrack: describeTrack(businessAudioSender?.track),
- aiTrackAttachedToBusiness,
- videoRecording: videoRecorder?.state === 'recording' ? {
- track: describeTrack(videoRecordingTrack),
- seconds: ((Date.now() - videoRecordingStartedAt) / 1000).toFixed(1),
- chunks: videoRecordingChunks.length,
- } : null,
- businessOutboundStats: { ...businessOutboundStats },
- aiState: aiStateLabel(),
- audio,
- video,
- businessSenders,
- };
- };
-
- async function connectAi(signalingUrl, assistantId) {
- if (!ensureUserTracksBeforeConnect()) {
- setStatus('disconnected');
- return false;
- }
- disconnectAi();
-
- const pcId = generatePcId();
- aiPc = new RTCPeerConnection({ iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] });
- aiPc.__aiNativeCopilotInternal = true;
- aiPc.onconnectionstatechange = () => log(`AI PC connectionState=${aiPc.connectionState}`);
- aiPc.oniceconnectionstatechange = () => log(`AI PC iceConnectionState=${aiPc.iceConnectionState}`);
-
- aiDc = aiPc.createDataChannel('chat');
- aiDc.onopen = () => {
- setStatus('connected');
- log('AI data channel open', true);
- aiDc.send(JSON.stringify({ type: 'client-ready' }));
- };
- aiDc.onclose = () => setStatus('disconnected');
- aiDc.onmessage = (event) => {
- try {
- const msg = JSON.parse(event.data);
- const content = msg.delta || msg.content;
- if (msg.type === 'transcript') {
- logUserTranscript(content);
- } else if (msg.type === 'assistant-text-start') {
- startAssistantLine();
- } else if (msg.type === 'assistant-text-delta') {
- appendAssistantDelta(content);
- } else if (msg.type === 'assistant-text-end') {
- finishAssistantLine();
- } else {
- log(`data channel message: ${msg.type}`);
- }
- } catch {
- appendChatLine('DATA', event.data);
- }
- };
-
- const sendAudio = shouldSendAudioToAi();
- const audioTransceiver = sendAudio && isUsableTrack(remoteUserAudioTrack)
- ? aiPc.addTransceiver(remoteUserAudioTrack, { direction: 'sendrecv' })
- : aiPc.addTransceiver('audio', { direction: sendAudio ? 'sendrecv' : 'recvonly' });
- aiAudioSender = audioTransceiver.sender;
- log(!sendAudio
- ? 'AI audio uplink disabled by panel setting'
- : isUsableTrack(remoteUserAudioTrack)
- ? 'AI input starts with native remote audio track'
- : 'AI input waiting for native remote audio track', true);
-
- const sendVideo = shouldSendVideoToAi();
- if (sendVideo) {
- const videoTransceiver = isUsableTrack(remoteUserVideoTrack)
- ? aiPc.addTransceiver(remoteUserVideoTrack, { direction: 'sendonly' })
- : aiPc.addTransceiver('video', { direction: 'sendonly' });
- aiVideoSender = videoTransceiver.sender;
- if (isUsableTrack(remoteUserVideoTrack)) log('AI video starts with native remote video track', true);
- } else {
- aiVideoSender = null;
- log('AI video uplink disabled by panel setting');
- }
- if (sendAudio && isUsableTrack(remoteUserAudioTrack)) scheduleAiInputHealthCheck(remoteUserAudioTrack);
-
- aiPc.ontrack = (event) => {
- if (event.track.kind !== 'audio') return;
- ignoredTrackIds.add(event.track.id);
- aiResponseAudioTrack = event.track;
- log('received native AI response audio track', true);
- attachAiOutputToBusiness();
- };
-
- aiPc.onicecandidate = (event) => {
- if (aiWs?.readyState !== WebSocket.OPEN) return;
- aiWs.send(JSON.stringify({
- type: 'ice-candidate',
- payload: {
- pc_id: pcId,
- candidate: event.candidate ? {
- candidate: event.candidate.candidate,
- sdpMid: event.candidate.sdpMid,
- sdpMLineIndex: event.candidate.sdpMLineIndex,
- } : null,
- },
- }));
- };
-
- setStatus('connecting');
- aiWs = new WebSocket(signalingUrl);
- aiWs.onopen = async () => {
- const offer = await aiPc.createOffer();
- await aiPc.setLocalDescription(offer);
- aiWs.send(JSON.stringify({
- type: 'offer',
- payload: {
- pc_id: pcId,
- sdp: aiPc.localDescription.sdp,
- type: aiPc.localDescription.type,
- assistant_id: assistantId,
- vision_enabled: sendVideo,
- },
- }));
- log('sent AI WebRTC offer', true);
- };
- aiWs.onmessage = async (event) => {
- try {
- const msg = JSON.parse(event.data);
- if (msg.type === 'answer') {
- await aiPc.setRemoteDescription({ type: 'answer', sdp: msg.payload.sdp });
- log(`AI WebRTC answer applied; ${aiStateLabel()}`, true);
- } else if (msg.type === 'ice-candidate' && msg.payload?.candidate) {
- await aiPc.addIceCandidate(msg.payload.candidate).catch(() => {});
- } else if (msg.type === 'error') {
- throw new Error(msg.payload?.message || 'server error');
- }
- } catch (error) {
- log(`signaling error: ${error.message || error}`);
- }
- };
- aiWs.onerror = () => {
- setStatus('failed');
- log('AI signaling websocket error');
- };
- aiWs.onclose = () => setStatus('disconnected');
-
- attachRemoteInputToAi();
- if (sendVideo) attachRemoteVideoToAi();
- return true;
- }
-
- function disconnectAi() {
- if (aiDc) {
- try { aiDc.close(); } catch {}
- aiDc = null;
- }
- if (aiPc) {
- try { aiPc.close(); } catch {}
- aiPc = null;
- }
- if (aiWs) {
- try { aiWs.close(); } catch {}
- aiWs = null;
- }
- aiAudioSender = null;
- aiVideoSender = null;
- aiResponseAudioTrack = null;
- if (inputRecorder?.state === 'recording') stopInputRecording();
- if (videoRecorder?.state === 'recording') stopVideoRecording();
- setStatus('disconnected');
- }
-
- function setStatus(status) {
- const el = document.getElementById('native-copilot-status');
- if (!el) return;
- const labels = {
- disconnected: '未连接',
- connecting: '连接中',
- connected: '已连接',
- failed: '连接失败',
- };
- el.textContent = labels[status] || status;
- }
-
- function updateModeButton() {
- const button = document.getElementById('native-copilot-toggle');
- if (!button) return;
- if (isAiTakeover) {
- button.textContent = '当前：AI 接管｜点击切回人工';
- button.style.background = '#8b5cf6';
- } else {
- button.textContent = '当前：人工接管｜点击切换 AI';
- button.style.background = '#10b981';
- }
- }
-
- function renderPanel() {
- if (!document.body || document.getElementById('native-copilot-panel')) return;
-
- const panel = document.createElement('div');
- panel.id = 'native-copilot-panel';
- panel.style.cssText = [
- 'position:fixed',
- 'top:20px',
- 'left:20px',
- 'width:360px',
- 'z-index:999999',
- 'box-sizing:border-box',
- 'padding:16px',
- 'border-radius:10px',
- 'background:rgba(17,24,39,.96)',
- 'color:white',
- 'font-family:system-ui,-apple-system,BlinkMacSystemFont,sans-serif',
- 'box-shadow:0 12px 32px rgba(0,0,0,.35)',
- ].join(';');
-
- panel.innerHTML = `
-
- AI 原生选轨
- 未连接
-
-
- 音频输入：布局优先自动选轨
- 视频输入：远端视频轨
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- 等待交互...
-
- `;
-
- document.body.appendChild(panel);
-
- const toggle = document.getElementById('native-copilot-toggle');
- const recordButton = document.getElementById('native-copilot-record');
- const videoRecordButton = document.getElementById('native-copilot-record-video');
- const urlInput = document.getElementById('native-copilot-url');
- const assistantInput = document.getElementById('native-copilot-assistant');
- const sendAudioInput = document.getElementById('native-copilot-send-audio');
- const sendVideoInput = document.getElementById('native-copilot-send-video');
-
- sendAudioInput.checked = shouldSendAudioToAi();
- sendVideoInput.checked = shouldSendVideoToAi();
- sendAudioInput.addEventListener('change', () => {
- writeBoolSetting(AUDIO_UPLINK_STORAGE_KEY, sendAudioInput.checked);
- log(`AI audio uplink setting: ${sendAudioInput.checked ? 'on' : 'off'}`);
- });
- sendVideoInput.addEventListener('change', () => {
- writeBoolSetting(VIDEO_UPLINK_STORAGE_KEY, sendVideoInput.checked);
- log(`AI video uplink setting: ${sendVideoInput.checked ? 'on' : 'off'}`);
- });
- recordButton.addEventListener('click', toggleInputRecording);
- videoRecordButton.addEventListener('click', toggleVideoRecording);
- updateRecordButton();
- updateVideoRecordButton();
- updateSelectedInputLabel();
- updateModeButton();
-
- toggle.addEventListener('click', async () => {
- if (!isAiTakeover && !assistantInput.value.trim()) {
- alert('请先填写助手 ID。');
- return;
- }
-
- if (!isAiTakeover && !ensureUserTracksBeforeConnect()) {
- updateModeButton();
- return;
- }
-
- isAiTakeover = !isAiTakeover;
- updateModeButton();
- if (isAiTakeover) {
- const connected = await connectAi(urlInput.value.trim(), assistantInput.value.trim());
- if (connected) await attachAiOutputToBusiness();
- else {
- isAiTakeover = false;
- updateModeButton();
- }
- } else {
- await restoreBusinessAudio();
- disconnectAi();
- }
- });
- }
-
- function renderPanelSoon() {
- try { renderPanel(); } catch (error) { console.warn('[AI Native Copilot] panel error', error); }
- }
-
- if (document.readyState === 'loading') {
- document.addEventListener('DOMContentLoaded', renderPanelSoon, { once: true });
- } else {
- renderPanelSoon();
- }
- requestAnimationFrame(renderVu);
- setInterval(refreshVuFromStats, 250);
- setTimeout(renderPanelSoon, 1500);
- setTimeout(renderPanelSoon, 3000);
-})();
diff --git a/scripts/copilot.js b/scripts/copilot.js
index c5d53f6..ab9cf9e 100644
--- a/scripts/copilot.js
+++ b/scripts/copilot.js
@@ -1,5 +1,5 @@
// ==UserScript==
-// @name 12345 AI 协同中台 (WebRTC 洗流桥接版)
+// @name 12345 AI 协同中台
// @namespace http://tampermonkey.net/
// @version 7.5
// @description 引入 PCM ScriptProcessor 强制软解防丢包Bug；增加真实洗流耳返监听功能，彻底解决远端触发失败悬案
@@ -23,6 +23,8 @@
}
const origAddEventListener = RTCPeerConnection.prototype.addEventListener;
const origSetRemoteDescription = RTCPeerConnection.prototype.setRemoteDescription;
+ const origAddTrack = RTCPeerConnection.prototype.addTrack;
+ const origAddTransceiver = RTCPeerConnection.prototype.addTransceiver;
const origAddStream = RTCPeerConnection.prototype.addStream;
const mediaSrcObjectDescriptor = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, 'srcObject');
@@ -43,6 +45,13 @@
let audioSender = null;
let videoSender = null;
let analyserNode = null;
+ let businessAudioSender = null;
+ let businessOriginalAudioTrack = null;
+ let aiRemoteAudioTrack = null;
+ let aiTrackAttachedToBusiness = false;
+ let aiInputMode = 'washed';
+ let directInputHealthTimer = null;
+ let directInputTrackId = null;
let isAiTakeover = false;
let isMonitoring = false;
@@ -59,6 +68,15 @@
const remoteReceiversByTrackId = new Map();
const receiverStatsMemory = new Map();
const capturedVideoElementStreams = new WeakMap();
+ function createAudioContext() {
+ const AudioContextClass = window.AudioContext || window.webkitAudioContext;
+ try {
+ return new AudioContextClass({ sampleRate: 48000 });
+ } catch (e) {
+ console.warn("48k AudioContext 创建失败，回退浏览器默认采样率:", e);
+ return new AudioContextClass();
+ }
+ }
function isInternalPeerConnection(target) {
return !!(target && target.__aiCopilotInternal);
@@ -98,10 +116,193 @@
console.log(`[AI Copilot WebRTC] 绑定远端 receiver(${reason})`, track.kind, track.id);
}
+ function rememberBusinessAudioSender(sender, track, reason = 'unknown') {
+ if (!sender) return;
+ const senderTrack = track || sender.track;
+ if (senderTrack && senderTrack.kind !== 'audio') return;
+ businessAudioSender = sender;
+ if (senderTrack && senderTrack !== aiRemoteAudioTrack) {
+ businessOriginalAudioTrack = senderTrack;
+ }
+ console.log(`[AI Copilot WebRTC] 绑定业务通话 audio sender(${reason})`, senderTrack?.id || 'no-track-yet');
+ if (isAiTakeover && aiRemoteAudioTrack) attachAiTrackToBusiness();
+ }
+
+ function wrapBusinessSender(sender) {
+ if (!sender || sender.__aiCopilotWrappedSender) return sender;
+ const originalReplaceTrack = sender.replaceTrack?.bind(sender);
+ if (!originalReplaceTrack) return sender;
+
+ sender.replaceTrack = async function(track) {
+ const ret = await originalReplaceTrack(track);
+ if (track && track.kind === 'audio' && track !== aiRemoteAudioTrack && !isAiTakeover) {
+ rememberBusinessAudioSender(sender, track, 'sender-replaceTrack');
+ }
+ return ret;
+ };
+ sender.__aiCopilotWrappedSender = true;
+ return sender;
+ }
+
+ async function attachAiTrackToBusiness() {
+ if (!isAiTakeover || !businessAudioSender || !aiRemoteAudioTrack || aiRemoteAudioTrack.readyState !== 'live') return false;
+ try {
+ await businessAudioSender.replaceTrack(aiRemoteAudioTrack);
+ aiTrackAttachedToBusiness = true;
+ appendLog("🔁 已将 AI 原生音轨直接替换到业务通话上行", true);
+ return true;
+ } catch (e) {
+ aiTrackAttachedToBusiness = false;
+ appendLog(`⚠️ AI 原生音轨替换业务 sender 失败: ${e.message || e}`);
+ return false;
+ }
+ }
+
+ async function restoreBusinessAudioTrack() {
+ if (!businessAudioSender || !aiTrackAttachedToBusiness) return;
+ try {
+ await businessAudioSender.replaceTrack(businessOriginalAudioTrack || null);
+ appendLog("👨💼 已恢复业务通话原始音轨", true);
+ } catch (e) {
+ appendLog(`⚠️ 恢复业务音轨失败: ${e.message || e}`);
+ } finally {
+ aiTrackAttachedToBusiness = false;
+ }
+ }
+
+ async function outboundAudioBytes(sender) {
+ if (!sender?.getStats) return null;
+ try {
+ let total = 0;
+ const report = await sender.getStats();
+ report.forEach(stat => {
+ const statKind = stat.kind || stat.mediaType;
+ if (stat.type === 'outbound-rtp' && statKind === 'audio' && typeof stat.bytesSent === 'number') {
+ total += stat.bytesSent;
+ }
+ });
+ return total;
+ } catch (e) {
+ return null;
+ }
+ }
+
+ function aiPeerReadyForMedia() {
+ if (!pc || !pc.remoteDescription) return false;
+ return (
+ pc.connectionState === 'connected'
+ || pc.iceConnectionState === 'connected'
+ || pc.iceConnectionState === 'completed'
+ );
+ }
+
+ function aiPeerStateLabel() {
+ if (!pc) return 'pc=null';
+ return `conn=${pc.connectionState},ice=${pc.iceConnectionState},signaling=${pc.signalingState},remote=${pc.remoteDescription?.type || 'none'}`;
+ }
+
+ function waitForAiPeerMediaReady(timeoutMs = 10000) {
+ if (aiPeerReadyForMedia()) return Promise.resolve(true);
+ return new Promise((resolve) => {
+ const startedAt = Date.now();
+ const timer = setInterval(() => {
+ if (aiPeerReadyForMedia()) {
+ clearInterval(timer);
+ resolve(true);
+ } else if (Date.now() - startedAt >= timeoutMs) {
+ clearInterval(timer);
+ resolve(false);
+ }
+ }, 250);
+ });
+ }
+
+ function disconnectWashedInputGraph() {
+ if (currentMeterSourceNode) {
+ try { currentMeterSourceNode.disconnect(); } catch(e) {}
+ currentMeterSourceNode = null;
+ }
+ if (currentVadBoostGainNode) {
+ try { currentVadBoostGainNode.disconnect(); } catch(e) {}
+ currentVadBoostGainNode = null;
+ }
+ if (pcmForcerNode) {
+ try { pcmForcerNode.disconnect(); } catch(e) {}
+ }
+ }
+
+ async function fallbackToWashedAiInput(track, reason = 'fallback') {
+ if (!track || track.readyState !== 'live') return;
+ if (directInputHealthTimer) {
+ clearTimeout(directInputHealthTimer);
+ directInputHealthTimer = null;
+ }
+ directInputTrackId = null;
+ if (audioSender && washedUploadTrack && audioSender.track !== washedUploadTrack) {
+ try {
+ await audioSender.replaceTrack(washedUploadTrack);
+ } catch (e) {
+ appendLog(`⚠️ AI 输入切回洗流轨失败: ${e.message || e}`);
+ }
+ }
+ aiInputMode = 'washed';
+ appendLog(`🧯 AI 输入切回洗流链路(${reason})`, true);
+ ensureVuMeterForTrack(track, reason);
+ }
+
+ async function tryDirectAiInput(track, reason = 'direct') {
+ if (!track || track.readyState !== 'live' || window.__ignoredTrackIds.has(track.id)) return false;
+ if (!audioSender) {
+ pendingBridgeAudioTrack = track;
+ return false;
+ }
+ if (aiInputMode === 'direct' && directInputTrackId === track.id && audioSender.track === track) return true;
+
+ try {
+ if (directInputHealthTimer) clearTimeout(directInputHealthTimer);
+ await audioSender.replaceTrack(track);
+ aiInputMode = 'direct';
+ directInputTrackId = track.id;
+ disconnectWashedInputGraph();
+ appendLog(`🚀 AI 输入直连远端原生音轨(${reason}); track=${track.id}, ${aiPeerStateLabel()}`, true);
+
+ directInputHealthTimer = setTimeout(async () => {
+ if (aiInputMode !== 'direct' || directInputTrackId !== track.id || audioSender?.track !== track) return;
+ appendLog(`🩺 等待 AI WebRTC 媒体连接后检查直连输入: ${aiPeerStateLabel()}`);
+ const ready = await waitForAiPeerMediaReady(10000);
+ if (aiInputMode !== 'direct' || directInputTrackId !== track.id || audioSender?.track !== track) return;
+ if (!ready) {
+ appendLog(`⚠️ AI WebRTC 媒体连接超时，直连输入无法验证: ${aiPeerStateLabel()}`);
+ await fallbackToWashedAiInput(track, 'direct-ai-pc-not-ready');
+ return;
+ }
+ const beforeHealthBytes = await outboundAudioBytes(audioSender);
+ await new Promise(resolve => setTimeout(resolve, 2500));
+ if (aiInputMode !== 'direct' || directInputTrackId !== track.id || audioSender?.track !== track) return;
+ const afterBytes = await outboundAudioBytes(audioSender);
+ appendLog(`🩺 直连输入发包检查: before=${beforeHealthBytes}, after=${afterBytes}, ${aiPeerStateLabel()}`);
+ if (
+ afterBytes !== null
+ && (
+ (beforeHealthBytes !== null && afterBytes <= beforeHealthBytes)
+ || (beforeHealthBytes === null && afterBytes === 0)
+ )
+ ) {
+ await fallbackToWashedAiInput(track, 'direct-no-outbound-audio');
+ }
+ }, 500);
+ return true;
+ } catch (e) {
+ appendLog(`⚠️ AI 输入直连失败，改用洗流: ${e.message || e}`);
+ await fallbackToWashedAiInput(track, 'direct-replace-failed');
+ return false;
+ }
+ }
+
function initMeterAudioCtx() {
if (meterAudioCtx) return;
try {
- meterAudioCtx = new (window.AudioContext || window.webkitAudioContext)();
+ meterAudioCtx = createAudioContext();
meterKeepAliveGainNode = meterAudioCtx.createGain();
meterKeepAliveGainNode.gain.value = 0;
@@ -134,14 +335,23 @@
}
}
- function bridgeRemoteAudioTrack(track, reason = 'scan') {
+ async function bridgeRemoteAudioTrack(track, reason = 'scan') {
if (!track || track.readyState !== 'live' || window.__ignoredTrackIds.has(track.id)) return;
- if (track === currentStolenAudioTrack && currentMeterSourceNode) return;
+ if (
+ track === currentStolenAudioTrack
+ && (
+ (aiInputMode === 'direct' && audioSender?.track === track)
+ || (aiInputMode === 'washed' && currentMeterSourceNode)
+ )
+ ) return;
pendingBridgeAudioTrack = track;
- appendLog(`🎙️ 锁定远端发声轨(${reason})，接入混流总线`, true);
+ appendLog(`🎙️ 锁定远端发声轨(${reason})，尝试直连 AI 输入`, true);
currentStolenAudioTrack = track;
- ensureVuMeterForTrack(track, reason);
+ const directOk = await tryDirectAiInput(track, reason);
+ if (!directOk && audioSender) {
+ await fallbackToWashedAiInput(track, `${reason}-fallback`);
+ }
}
function capturePlayableElementVideoStream(el) {
@@ -196,7 +406,7 @@
// 2. AGC 放大增益 (击穿 VAD 阈值)
const vadBoostGain = meterAudioCtx.createGain();
- vadBoostGain.gain.value = 8.0; // 放大 8 倍
+ vadBoostGain.gain.value = 2.0; // fallback 洗流仅轻度放大，避免底噪/回声被过度抬高
currentVadBoostGainNode = vadBoostGain;
currentMeterSourceNode.connect(vadBoostGain);
@@ -377,6 +587,28 @@
return origAddEventListener.call(this, type, listener, options);
};
+ RTCPeerConnection.prototype.addTrack = function(track, ...streams) {
+ const sender = origAddTrack.call(this, track, ...streams);
+ if (!isInternalPeerConnection(this) && track?.kind === 'audio') {
+ rememberBusinessAudioSender(wrapBusinessSender(sender), track, 'pc-addTrack');
+ }
+ return sender;
+ };
+
+ if (origAddTransceiver) {
+ RTCPeerConnection.prototype.addTransceiver = function(trackOrKind, init) {
+ const transceiver = origAddTransceiver.call(this, trackOrKind, init);
+ if (!isInternalPeerConnection(this)) {
+ const track = trackOrKind instanceof MediaStreamTrack ? trackOrKind : init?.sendEncodings ? transceiver.sender?.track : null;
+ const kind = trackOrKind instanceof MediaStreamTrack ? trackOrKind.kind : trackOrKind;
+ if (kind === 'audio') {
+ rememberBusinessAudioSender(wrapBusinessSender(transceiver.sender), track || transceiver.sender?.track, 'pc-addTransceiver');
+ }
+ }
+ return transceiver;
+ };
+ }
+
try {
const onTrackDescriptor = Object.getOwnPropertyDescriptor(RTCPeerConnection.prototype, 'ontrack');
Object.defineProperty(RTCPeerConnection.prototype, 'ontrack', {
@@ -526,7 +758,7 @@
}
bridgeRemoteVideoTrack(foundVideoTrack);
- bridgeRemoteAudioTrack(foundAudioTrack, 'scan');
+ bridgeRemoteAudioTrack(foundAudioTrack, 'scan').catch(e => console.warn(e));
}, 1000);
// ==========================================
@@ -544,39 +776,20 @@
realStream.__internal_bypass = true;
realStream.getTracks().forEach(t => window.__ignoredTrackIds.add(t.id));
- audioCtx = new (window.AudioContext || window.webkitAudioContext)();
-
- const mixedDest = audioCtx.createMediaStreamDestination();
- mixedDest.stream.__internal_bypass = true;
- mixedDest.stream.getTracks().forEach(t => window.__ignoredTrackIds.add(t.id));
-
- if (realStream.getAudioTracks().length > 0) {
- const localMicSource = audioCtx.createMediaStreamSource(realStream);
-
- micGainNode = audioCtx.createGain();
- micGainNode.gain.value = isAiTakeover ? 0 : 1.0;
- localMicSource.connect(micGainNode);
- micGainNode.connect(mixedDest);
- }
-
- aiOutGainNode = audioCtx.createGain();
- aiOutGainNode.gain.value = isAiTakeover ? 1.0 : 0;
- aiOutGainNode.connect(mixedDest);
+ audioCtx = createAudioContext();
+ micGainNode = null;
+ aiOutGainNode = null;
aiLocalGainNode = audioCtx.createGain();
aiLocalGainNode.gain.value = isAiTakeover ? 1.0 : 0;
aiLocalGainNode.connect(audioCtx.destination);
- if (constraints.video && realStream.getVideoTracks().length > 0) {
- mixedDest.stream.addTrack(realStream.getVideoTracks()[0]);
- }
-
if (pendingBridgeAudioTrack && pendingBridgeAudioTrack.readyState === 'live') {
- bridgeRemoteAudioTrack(pendingBridgeAudioTrack, 'pending');
+ bridgeRemoteAudioTrack(pendingBridgeAudioTrack, 'pending').catch(e => console.warn(e));
}
renderControlPanelSoon();
- return mixedDest.stream;
+ return realStream;
} catch (err) {
console.error("🚨 拦截失败:", err);
return originalGetUserMedia(constraints);
@@ -598,6 +811,9 @@
if(statusEl) statusEl.textContent = "🟡 协商中...";
appendLog("🔄 正在建立 WebRTC 连接...");
+ pc.onconnectionstatechange = () => appendLog(`🔌 AI PC connectionState=${pc.connectionState}`);
+ pc.oniceconnectionstatechange = () => appendLog(`🧊 AI PC iceConnectionState=${pc.iceConnectionState}`);
+
initMeterAudioCtx();
const activeStream = new MediaStream([washedUploadTrack]);
const audioTransceiver = pc.addTransceiver(washedUploadTrack, {
@@ -605,7 +821,10 @@
streams: [activeStream]
});
audioSender = audioTransceiver.sender;
- appendLog("🎤 已挂载洗流音频管线 (附带 PCM 强刷保障)", true);
+ appendLog("🎤 已挂载 AI 输入音频通道 (优先直连，失败回退洗流)", true);
+ if (pendingBridgeAudioTrack && pendingBridgeAudioTrack.readyState === 'live') {
+ bridgeRemoteAudioTrack(pendingBridgeAudioTrack, 'sender-ready').catch(e => console.warn(e));
+ }
const videoTransceiver = pc.addTransceiver('video', { direction: 'sendonly' });
videoSender = videoTransceiver.sender;
@@ -615,16 +834,17 @@
pc.ontrack = (e) => {
if (e.track.kind === 'audio' && audioCtx) {
- appendLog("🎧 接收到原生 AI 语音流，接入本地拓扑", true);
+ appendLog("🎧 接收到原生 AI 语音流，准备直连业务通话", true);
window.__ignoredTrackIds.add(e.track.id);
+ aiRemoteAudioTrack = e.track;
const aiStream = new MediaStream([e.track]);
aiStream.__internal_bypass = true;
const aiStreamSource = audioCtx.createMediaStreamSource(aiStream);
- aiStreamSource.connect(aiOutGainNode);
- aiStreamSource.connect(aiLocalGainNode);
+ if (aiLocalGainNode) aiStreamSource.connect(aiLocalGainNode);
window.__aiStreamSource = aiStreamSource;
+ attachAiTrackToBusiness();
}
};
@@ -715,6 +935,12 @@
}
function disconnectWebRTC() {
+ if (directInputHealthTimer) {
+ clearTimeout(directInputHealthTimer);
+ directInputHealthTimer = null;
+ }
+ aiInputMode = 'washed';
+ directInputTrackId = null;
if (dc) {
try { dc.close(); } catch(e){}
dc = null;
@@ -758,7 +984,7 @@
-
+
@@ -891,7 +1117,7 @@
fileInput.click();
});
- btnToggleTakeover.addEventListener('click', () => {
+ btnToggleTakeover.addEventListener('click', async () => {
if (!audioCtx) {
alert("音频路由未初始化！请先在网页内加入会议。");
return;
@@ -907,8 +1133,6 @@
const now = audioCtx.currentTime;
if (isAiTakeover) {
- if(micGainNode) micGainNode.gain.setValueAtTime(0, now);
- if(aiOutGainNode) aiOutGainNode.gain.setValueAtTime(1.0, now);
if(aiLocalGainNode) aiLocalGainNode.gain.setValueAtTime(1.0, now);
btnToggleTakeover.textContent = "🤖 AI 托管中 (点击切回人工)";
@@ -921,10 +1145,13 @@
}
connectWebRTC(urlInput.value, asstId);
}
+ if (!businessAudioSender) {
+ appendLog("⚠️ 尚未捕获业务通话 audio sender，等待网页建立上行轨道");
+ }
+ await attachAiTrackToBusiness();
} else {
- if(micGainNode) micGainNode.gain.setValueAtTime(1.0, now);
- if(aiOutGainNode) aiOutGainNode.gain.setValueAtTime(0, now);
if(aiLocalGainNode) aiLocalGainNode.gain.setValueAtTime(0, now);
+ await restoreBusinessAudioTrack();
btnToggleTakeover.textContent = "👨💼 人工通话中 (点击 AI 托管)";
btnToggleTakeover.style.background = "#10b981";