diff --git a/scripts/copilot-native-element.js b/scripts/copilot-native-element.js deleted file mode 100644 index 61a7042..0000000 --- a/scripts/copilot-native-element.js +++ /dev/null @@ -1,1006 +0,0 @@ -// ==UserScript== -// @name 12345 AI Native Element Copilot -// @namespace http://tampermonkey.net/ -// @version 1.0 -// @description Native-track bridge with fixed element selection: element-2:audio input, element-0:video vision. -// @author You -// @match *://yhy-yw.22sj.cn:*/* -// @grant none -// @run-at document-start -// ==/UserScript== - -(function () { - 'use strict'; - - const DEFAULT_SIGNALING_URL = 'ws://182.92.86.220:8000/ws/voice'; - const STATIC_AUDIO_ELEMENT_REASON = 'element-2:audio'; - const STATIC_VIDEO_ELEMENT_REASON = 'element-0:video'; - - const originalAddEventListener = RTCPeerConnection.prototype.addEventListener; - const originalAddTrack = RTCPeerConnection.prototype.addTrack; - const originalAddTransceiver = RTCPeerConnection.prototype.addTransceiver; - const originalSetRemoteDescription = RTCPeerConnection.prototype.setRemoteDescription; - const mediaSrcObjectDescriptor = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, 'srcObject'); - - const remoteAudio = new Map(); - const remoteVideo = new Map(); - const businessSenders = new Map(); - const aiOutputTrackIds = new Set(); - const inboundMemory = new Map(); - const outboundMemory = new Map(); - const internalPcs = new WeakSet(); - const AUDIO_UPLINK_STORAGE_KEY = 'element-copilot-send-audio'; - const VIDEO_UPLINK_STORAGE_KEY = 'element-copilot-send-video'; - - let selectedRemoteAudio = null; - let selectedRemoteVideo = null; - let selectedBusinessSender = null; - let originalBusinessTrack = null; - let aiAudioTrack = null; - - let aiPc = null; - let aiWs = null; - let aiDc = null; - let aiAudioSender = null; - let aiVideoSender = null; - let isAiTakeover = false; - let aiAttached = false; - let inputVu = 0; - let outputVu = 0; - let inputHealthTimer = null; - let outputHealthTimer = null; - let assistantLine = null; - let assistantBuffer = ''; - let inputRecorder = null; - let inputRecordingChunks = []; - let inputRecordingTrack = null; - let inputRecordingStartedAt = 0; - - function log(message) { - console.log(`[AI Native Element] ${message}`); - } - - function readBoolSetting(key, fallback) { - try { - const value = localStorage.getItem(key); - if (value === null) return fallback; - return value === 'true'; - } catch { - return fallback; - } - } - - function writeBoolSetting(key, value) { - try { - localStorage.setItem(key, value ? 'true' : 'false'); - } catch { - // Ignore storage failures; the current checkbox state still applies. - } - } - - function shouldSendAudioToAi() { - return readBoolSetting(AUDIO_UPLINK_STORAGE_KEY, true); - } - - function shouldSendVideoToAi() { - return readBoolSetting(VIDEO_UPLINK_STORAGE_KEY, true); - } - - function appendChatLine(role, text, important = false) { - const box = document.getElementById('element-copilot-log'); - if (!box) return; - if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = ''; - const row = document.createElement('div'); - row.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] ${role}: ${text}`; - row.style.marginBottom = '4px'; - if (important) row.style.color = '#fcd34d'; - box.appendChild(row); - while (box.children.length > 120) box.removeChild(box.firstChild); - box.scrollTop = box.scrollHeight; - return row; - } - - function startAssistantLine() { - assistantBuffer = ''; - assistantLine = null; - } - - function appendAssistantDelta(delta) { - if (!delta) return; - const box = document.getElementById('element-copilot-log'); - if (!box) { - console.log(`[AI Native Element] AI: ${delta}`); - return; - } - if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = ''; - if (!assistantLine) assistantLine = appendChatLine('AI', ''); - assistantBuffer += delta; - assistantLine.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] AI: ${assistantBuffer}`; - box.scrollTop = box.scrollHeight; - } - - function finishAssistantLine() { - assistantLine = null; - assistantBuffer = ''; - } - - function logUserTranscript(text) { - if (!text?.trim()) return; - finishAssistantLine(); - appendChatLine('用户', text.trim(), true); - } - - function recorderMimeType() { - if (window.MediaRecorder?.isTypeSupported?.('audio/webm;codecs=opus')) return 'audio/webm;codecs=opus'; - if (window.MediaRecorder?.isTypeSupported?.('audio/webm')) return 'audio/webm'; - return ''; - } - - function recordingFileName(track) { - const stamp = new Date().toISOString().replace(/[:.]/g, '-'); - const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track'; - return `ai-input-${trackPart}-${stamp}.webm`; - } - - async function startInputRecording() { - if (inputRecorder?.state === 'recording') return; - selectRemoteAudio('record'); - if (!isLive(selectedRemoteAudio)) { - appendChatLine('录音', `没有可录制的 ${STATIC_AUDIO_ELEMENT_REASON} 音轨`, true); - return; - } - if (!window.MediaRecorder) { - appendChatLine('录音', '当前浏览器不支持 MediaRecorder', true); - return; - } - - inputRecordingChunks = []; - inputRecordingTrack = selectedRemoteAudio; - inputRecordingStartedAt = Date.now(); - const stream = new MediaStream([selectedRemoteAudio]); - const mimeType = recorderMimeType(); - inputRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); - inputRecorder.ondataavailable = (event) => { - if (event.data?.size > 0) inputRecordingChunks.push(event.data); - }; - inputRecorder.onstop = () => { - const blob = new Blob(inputRecordingChunks, { type: inputRecorder?.mimeType || 'audio/webm' }); - const url = URL.createObjectURL(blob); - const link = document.createElement('a'); - link.href = url; - link.download = recordingFileName(inputRecordingTrack); - document.body.appendChild(link); - link.click(); - link.remove(); - setTimeout(() => URL.revokeObjectURL(url), 30000); - - const seconds = ((Date.now() - inputRecordingStartedAt) / 1000).toFixed(1); - appendChatLine('录音', `已保存 ${link.download}（${seconds}s，${(blob.size / 1024).toFixed(1)} KB）`, true); - inputRecorder = null; - inputRecordingChunks = []; - inputRecordingTrack = null; - updateRecordButton(); - }; - inputRecorder.start(250); - appendChatLine('录音', `开始录制 AI 输入：${trackLabel(inputRecordingTrack)}`, true); - updateRecordButton(); - } - - function stopInputRecording() { - if (inputRecorder?.state === 'recording') inputRecorder.stop(); - } - - function toggleInputRecording() { - if (inputRecorder?.state === 'recording') stopInputRecording(); - else startInputRecording(); - } - - function updateRecordButton() { - const button = document.getElementById('element-copilot-record'); - if (!button) return; - const recording = inputRecorder?.state === 'recording'; - button.textContent = recording ? '停止录制 AI 输入' : '录制 AI 输入'; - button.style.background = recording ? '#dc2626' : '#2563eb'; - } - - function pcId() { - const bytes = new Uint8Array(16); - crypto.getRandomValues(bytes); - return 'PC-' + Array.from(bytes).map((b) => b.toString(16).padStart(2, '0')).join(''); - } - - function isInternalPc(pc) { - return Boolean(pc && (pc.__aiNativeElementInternal || internalPcs.has(pc) || pc === aiPc)); - } - - function isLive(track) { - return track && track.readyState === 'live'; - } - - function trackLabel(track) { - if (!track) return 'none'; - return `${track.id}${track.muted ? ' muted' : ''}`; - } - - function markInternalPc(pc) { - if (!pc) return; - pc.__aiNativeElementInternal = true; - internalPcs.add(pc); - } - - function elementReasonFor(element) { - if (!element) return null; - const tag = element.tagName?.toLowerCase(); - if (tag !== 'audio' && tag !== 'video') return null; - const elements = Array.from(document.querySelectorAll(tag)); - const index = elements.indexOf(element); - if (index < 0) return `${tag}-srcObject`; - return `element-${index}:${tag}`; - } - - function hasElementReason(item, reason) { - return Boolean(item?.reasons?.has(reason)); - } - - function rememberRemoteReceiver(receiver, reason) { - const track = receiver?.track; - if (!isLive(track) || isInternalPc(receiver?.transport)) return; - - if (track.kind === 'audio' && !aiOutputTrackIds.has(track.id)) { - let item = remoteAudio.get(track.id); - if (!item) { - item = { track, receiver, reasons: new Set(), bytesDelta: 0, packetsDelta: 0 }; - remoteAudio.set(track.id, item); - log(`audio candidate ${trackLabel(track)} (${reason})`, true); - } - item.track = track; - item.receiver = receiver || item.receiver; - item.reasons.add(reason); - selectRemoteAudio(reason); - } - - if (track.kind === 'video') { - let item = remoteVideo.get(track.id); - if (!item) { - item = { track, receiver, reasons: new Set() }; - remoteVideo.set(track.id, item); - log(`video candidate ${trackLabel(track)} (${reason})`, true); - } - item.track = track; - item.receiver = receiver || item.receiver; - item.reasons.add(reason); - selectRemoteVideo(reason); - } - } - - function rememberRemoteStream(stream, reason) { - if (!stream || !(stream instanceof MediaStream)) return; - for (const track of stream.getAudioTracks()) { - if (aiOutputTrackIds.has(track.id)) continue; - let item = remoteAudio.get(track.id); - if (!item) { - item = { track, receiver: null, reasons: new Set(), bytesDelta: 0, packetsDelta: 0 }; - remoteAudio.set(track.id, item); - log(`audio stream candidate ${trackLabel(track)} (${reason})`, true); - } - item.track = track; - item.reasons.add(reason); - } - for (const track of stream.getVideoTracks()) { - let item = remoteVideo.get(track.id); - if (!item) { - item = { track, receiver: null, reasons: new Set() }; - remoteVideo.set(track.id, item); - log(`video stream candidate ${trackLabel(track)} (${reason})`, true); - } - item.track = track; - item.reasons.add(reason); - } - selectRemoteAudio(reason); - selectRemoteVideo(reason); - } - - function selectRemoteAudio(reason) { - const current = selectedRemoteAudio && remoteAudio.get(selectedRemoteAudio.id); - if (isLive(selectedRemoteAudio) && !selectedRemoteAudio.muted && hasElementReason(current, STATIC_AUDIO_ELEMENT_REASON)) { - return selectedRemoteAudio; - } - - const item = Array.from(remoteAudio.values()).find((candidate) => ( - isLive(candidate.track) - && !candidate.track.muted - && hasElementReason(candidate, STATIC_AUDIO_ELEMENT_REASON) - )); - if (!item) return null; - - if (selectedRemoteAudio !== item.track) { - selectedRemoteAudio = item.track; - log(`selected input audio (${reason}) from ${STATIC_AUDIO_ELEMENT_REASON}: ${trackLabel(selectedRemoteAudio)}`, true); - attachAudioToAi(); - } - return selectedRemoteAudio; - } - - function selectRemoteVideo(reason) { - const current = selectedRemoteVideo && remoteVideo.get(selectedRemoteVideo.id); - if (isLive(selectedRemoteVideo) && !selectedRemoteVideo.muted && hasElementReason(current, STATIC_VIDEO_ELEMENT_REASON)) { - return selectedRemoteVideo; - } - - const item = Array.from(remoteVideo.values()).find((candidate) => ( - isLive(candidate.track) - && !candidate.track.muted - && hasElementReason(candidate, STATIC_VIDEO_ELEMENT_REASON) - )); - if (!item) return null; - - if (selectedRemoteVideo !== item.track) { - selectedRemoteVideo = item.track; - log(`selected input video (${reason}) from ${STATIC_VIDEO_ELEMENT_REASON}: ${trackLabel(selectedRemoteVideo)}`, true); - attachVideoToAi(); - } - return selectedRemoteVideo; - } - - function rememberBusinessSender(sender, track, reason) { - if (!sender) return; - const senderTrack = track || sender.track; - if (senderTrack && senderTrack.kind !== 'audio') return; - - const id = sender.__aiNativeElementSenderId || `sender-${businessSenders.size + 1}`; - sender.__aiNativeElementSenderId = id; - - let item = businessSenders.get(id); - if (!item) { - item = { id, sender, track: senderTrack || null, originalTrack: senderTrack || null, sources: new Set(), bytesDelta: 0, packetsDelta: 0 }; - businessSenders.set(id, item); - log(`business sender candidate ${id} (${reason}): ${trackLabel(senderTrack)}`, true); - } - item.sender = sender; - item.track = sender.track || senderTrack || item.track; - if (!isAiTakeover && senderTrack && senderTrack !== aiAudioTrack) item.originalTrack = senderTrack; - item.sources.add(reason); - wrapBusinessSender(sender); - selectBusinessSender(reason); - } - - function wrapBusinessSender(sender) { - if (!sender || sender.__aiNativeElementWrapped) return; - const originalReplaceTrack = sender.replaceTrack?.bind(sender); - if (!originalReplaceTrack) return; - - sender.replaceTrack = async function (track) { - const result = await originalReplaceTrack(track); - rememberBusinessSender(sender, track, 'replaceTrack'); - return result; - }; - sender.__aiNativeElementWrapped = true; - } - - function isRemoteInputTrack(track) { - return Boolean(track && remoteAudio.has(track.id) && !aiOutputTrackIds.has(track.id)); - } - - function selectBusinessSender(reason) { - if (selectedBusinessSender && !isRemoteInputTrack(selectedBusinessSender.track)) return selectedBusinessSender; - - const item = Array.from(businessSenders.values()).find((candidate) => { - const track = candidate.sender.track || candidate.track || candidate.originalTrack; - return candidate.sources.has('pc-addTrack') && !isRemoteInputTrack(track); - }); - if (!item) return null; - - selectedBusinessSender = item.sender; - originalBusinessTrack = item.originalTrack || item.sender.track || originalBusinessTrack; - log(`selected output sender (${reason}): ${item.id}, track=${trackLabel(item.sender.track)}`, true); - attachAiOutputToBusiness(); - return selectedBusinessSender; - } - - function setVu(id, level) { - const el = document.getElementById(id); - if (el) el.style.width = `${Math.min(100, level * 2500)}%`; - } - - function renderVu() { - inputVu *= 0.88; - outputVu *= 0.88; - setVu('element-copilot-input-vu', inputVu); - setVu('element-copilot-output-vu', outputVu); - requestAnimationFrame(renderVu); - } - - async function refreshStats() { - await refreshInputStats(); - await refreshOutputStats(); - } - - async function refreshInputStats() { - for (const item of remoteAudio.values()) { - if (!item.receiver?.getStats) continue; - const report = await item.receiver.getStats(); - report.forEach((stat) => { - const kind = stat.kind || stat.mediaType; - if (stat.type !== 'inbound-rtp' || kind !== 'audio') return; - const prev = inboundMemory.get(stat.id); - const bytes = stat.bytesReceived || 0; - const packets = stat.packetsReceived || 0; - item.bytesReceived = bytes; - item.packetsReceived = packets; - item.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0; - item.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0; - inboundMemory.set(stat.id, { bytes, packets }); - if (item.track === selectedRemoteAudio) { - inputVu = Math.max(inputVu, Math.min(0.12, item.bytesDelta / 12000), Math.min(0.04, item.packetsDelta / 500)); - } - }); - } - } - - async function refreshOutputStats() { - for (const item of businessSenders.values()) { - if (!item.sender?.getStats) continue; - item.track = item.sender.track || item.track; - const report = await item.sender.getStats(); - report.forEach((stat) => { - const kind = stat.kind || stat.mediaType; - if (stat.type !== 'outbound-rtp' || kind !== 'audio') return; - const key = `${item.id}:${stat.id}`; - const prev = outboundMemory.get(key); - const bytes = stat.bytesSent || 0; - const packets = stat.packetsSent || 0; - item.bytesSent = bytes; - item.packetsSent = packets; - item.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0; - item.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0; - outboundMemory.set(key, { bytes, packets }); - if (item.sender === selectedBusinessSender) { - outputVu = Math.max(outputVu, Math.min(0.12, item.bytesDelta / 12000), Math.min(0.04, item.packetsDelta / 500)); - } - }); - } - } - - async function outboundBytes(sender) { - if (!sender?.getStats) return null; - let total = 0; - const report = await sender.getStats(); - report.forEach((stat) => { - const kind = stat.kind || stat.mediaType; - if (stat.type === 'outbound-rtp' && kind === 'audio') total += stat.bytesSent || 0; - }); - return total; - } - - function aiState() { - if (!aiPc) return 'pc=null'; - return `conn=${aiPc.connectionState},ice=${aiPc.iceConnectionState},signaling=${aiPc.signalingState},remote=${aiPc.remoteDescription?.type || 'none'}`; - } - - function aiReady() { - return Boolean(aiPc?.remoteDescription && ( - aiPc.connectionState === 'connected' - || aiPc.iceConnectionState === 'connected' - || aiPc.iceConnectionState === 'completed' - )); - } - - function waitForAiReady(timeoutMs = 10000) { - if (aiReady()) return Promise.resolve(true); - return new Promise((resolve) => { - const startedAt = Date.now(); - const timer = setInterval(() => { - if (aiReady()) { - clearInterval(timer); - resolve(true); - } else if (Date.now() - startedAt >= timeoutMs) { - clearInterval(timer); - resolve(false); - } - }, 250); - }); - } - - function checkAiInput(track) { - if (inputHealthTimer) clearTimeout(inputHealthTimer); - inputHealthTimer = setTimeout(async () => { - if (!aiAudioSender || aiAudioSender.track !== track) return; - await waitForAiReady(); - const before = await outboundBytes(aiAudioSender); - await new Promise((resolve) => setTimeout(resolve, 2500)); - const after = await outboundBytes(aiAudioSender); - log(`AI input packet check: before=${before}, after=${after}, ${aiState()}`, true); - }, 500); - } - - function checkAiOutput(track) { - if (outputHealthTimer) clearTimeout(outputHealthTimer); - outputHealthTimer = setTimeout(async () => { - if (!selectedBusinessSender || selectedBusinessSender.track !== track) return; - const before = await outboundBytes(selectedBusinessSender); - await new Promise((resolve) => setTimeout(resolve, 2500)); - const after = await outboundBytes(selectedBusinessSender); - log(`AI output packet check: before=${before}, after=${after}, sender=${trackLabel(selectedBusinessSender.track)}`, true); - }, 800); - } - - async function attachAudioToAi() { - if (!shouldSendAudioToAi()) return false; - if (!aiAudioSender || !isLive(selectedRemoteAudio)) return false; - if (aiAudioSender.track === selectedRemoteAudio) return true; - await aiAudioSender.replaceTrack(selectedRemoteAudio); - log(`AI input uses ${trackLabel(selectedRemoteAudio)} from ${STATIC_AUDIO_ELEMENT_REASON}`, true); - checkAiInput(selectedRemoteAudio); - return true; - } - - async function attachVideoToAi() { - if (!shouldSendVideoToAi()) return false; - if (!aiVideoSender || !isLive(selectedRemoteVideo)) return false; - if (aiVideoSender.track === selectedRemoteVideo) return true; - await aiVideoSender.replaceTrack(selectedRemoteVideo); - log(`AI video uses ${trackLabel(selectedRemoteVideo)} from ${STATIC_VIDEO_ELEMENT_REASON}`, true); - return true; - } - - async function attachAiOutputToBusiness() { - selectBusinessSender('before-output'); - if (!isAiTakeover || !selectedBusinessSender || !isLive(aiAudioTrack)) { - log(`AI output waiting: takeover=${isAiTakeover}, sender=${Boolean(selectedBusinessSender)}, aiTrack=${trackLabel(aiAudioTrack)}`); - return false; - } - if (selectedBusinessSender.track === aiAudioTrack) return true; - await selectedBusinessSender.replaceTrack(aiAudioTrack); - aiAttached = true; - log(`business sender uses AI audio ${trackLabel(aiAudioTrack)}`, true); - checkAiOutput(aiAudioTrack); - return true; - } - - async function restoreBusinessAudio() { - if (!selectedBusinessSender || !aiAttached) return; - await selectedBusinessSender.replaceTrack(originalBusinessTrack || null); - aiAttached = false; - log(`business sender restored ${trackLabel(originalBusinessTrack)}`, true); - } - - function wrapTrackListener(listener) { - if (!listener || listener.__aiNativeElementWrapped) return listener; - const wrapped = function (event) { - const ownerPc = this instanceof RTCPeerConnection ? this : event?.currentTarget; - if (!isInternalPc(ownerPc) && event?.track) { - rememberRemoteReceiver(event.receiver, 'pc-track'); - for (const stream of event.streams || []) rememberRemoteStream(stream, 'pc-track-stream'); - } - if (typeof listener === 'function') return listener.apply(this, arguments); - if (listener?.handleEvent) return listener.handleEvent(event); - return undefined; - }; - wrapped.__aiNativeElementWrapped = true; - return wrapped; - } - - RTCPeerConnection.prototype.addEventListener = function (type, listener, options) { - if (type === 'track') return originalAddEventListener.call(this, type, wrapTrackListener(listener), options); - return originalAddEventListener.call(this, type, listener, options); - }; - - try { - const descriptor = Object.getOwnPropertyDescriptor(RTCPeerConnection.prototype, 'ontrack'); - Object.defineProperty(RTCPeerConnection.prototype, 'ontrack', { - configurable: true, - enumerable: true, - get() { - return descriptor?.get ? descriptor.get.call(this) : this.__aiNativeElementOnTrack || null; - }, - set(listener) { - const wrapped = wrapTrackListener(listener); - this.__aiNativeElementOnTrack = wrapped; - if (descriptor?.set) descriptor.set.call(this, wrapped); - else this.addEventListener('track', wrapped); - }, - }); - } catch (error) { - console.warn('[AI Native Element] failed to wrap ontrack', error); - } - - RTCPeerConnection.prototype.addTrack = function (track, ...streams) { - const sender = originalAddTrack.call(this, track, ...streams); - if (!isInternalPc(this) && track?.kind === 'audio') rememberBusinessSender(sender, track, 'pc-addTrack'); - return sender; - }; - - if (originalAddTransceiver) { - RTCPeerConnection.prototype.addTransceiver = function (trackOrKind, init) { - const transceiver = originalAddTransceiver.call(this, trackOrKind, init); - if (!isInternalPc(this)) { - const kind = trackOrKind instanceof MediaStreamTrack ? trackOrKind.kind : trackOrKind; - if (kind === 'audio') { - const track = trackOrKind instanceof MediaStreamTrack ? trackOrKind : transceiver.sender?.track; - rememberBusinessSender(transceiver.sender, track, 'pc-addTransceiver'); - } - } - return transceiver; - }; - } - - RTCPeerConnection.prototype.setRemoteDescription = function () { - const result = originalSetRemoteDescription.apply(this, arguments); - if (!isInternalPc(this)) { - Promise.resolve(result).then(() => { - for (const receiver of this.getReceivers?.() || []) rememberRemoteReceiver(receiver, 'receiver-after-srd'); - for (const sender of this.getSenders?.() || []) rememberBusinessSender(sender, sender.track, 'sender-after-srd'); - }).catch(() => {}); - } - return result; - }; - - if (mediaSrcObjectDescriptor?.set) { - Object.defineProperty(HTMLMediaElement.prototype, 'srcObject', { - configurable: true, - enumerable: mediaSrcObjectDescriptor.enumerable, - get() { - return mediaSrcObjectDescriptor.get.call(this); - }, - set(stream) { - if (!this.__aiNativeElementCopilotElement) { - const reason = elementReasonFor(this) || `${this.tagName.toLowerCase()}-srcObject`; - rememberRemoteStream(stream, reason); - } - return mediaSrcObjectDescriptor.set.call(this, stream); - }, - }); - } - - function scanMediaElements() { - document.querySelectorAll('audio').forEach((element, index) => { - if (element.__aiNativeElementCopilotElement) return; - rememberRemoteStream(element.srcObject, `element-${index}:audio`); - }); - document.querySelectorAll('video').forEach((element, index) => { - if (element.__aiNativeElementCopilotElement) return; - rememberRemoteStream(element.srcObject, `element-${index}:video`); - }); - selectRemoteAudio('media-scan'); - selectRemoteVideo('media-scan'); - } - - setInterval(scanMediaElements, 1000); - - async function connectAi(signalingUrl, assistantId) { - disconnectAi(); - - const id = pcId(); - aiPc = new RTCPeerConnection({ iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] }); - markInternalPc(aiPc); - aiPc.onconnectionstatechange = () => log(`AI PC connectionState=${aiPc.connectionState}`); - aiPc.oniceconnectionstatechange = () => log(`AI PC iceConnectionState=${aiPc.iceConnectionState}`); - - aiDc = aiPc.createDataChannel('chat'); - aiDc.onopen = () => { - setStatus('connected'); - log('AI data channel open', true); - aiDc.send(JSON.stringify({ type: 'client-ready' })); - }; - aiDc.onclose = () => setStatus('disconnected'); - aiDc.onmessage = (event) => { - try { - const msg = JSON.parse(event.data); - const content = msg.delta || msg.content; - if (msg.type === 'transcript') { - logUserTranscript(content); - } else if (msg.type === 'assistant-text-start') { - startAssistantLine(); - } else if (msg.type === 'assistant-text-delta') { - appendAssistantDelta(content); - } else if (msg.type === 'assistant-text-end') { - finishAssistantLine(); - } else { - log(`data channel message: ${msg.type}`); - } - } catch { - appendChatLine('DATA', event.data); - } - }; - - scanMediaElements(); - selectRemoteAudio('connect'); - selectRemoteVideo('connect'); - selectBusinessSender('connect'); - - const sendAudio = shouldSendAudioToAi(); - const audioTransceiver = sendAudio && isLive(selectedRemoteAudio) - ? aiPc.addTransceiver(selectedRemoteAudio, { direction: 'sendrecv' }) - : aiPc.addTransceiver('audio', { direction: sendAudio ? 'sendrecv' : 'recvonly' }); - aiAudioSender = audioTransceiver.sender; - log(!sendAudio - ? 'AI audio uplink disabled by panel setting' - : isLive(selectedRemoteAudio) - ? `AI input starts with ${trackLabel(selectedRemoteAudio)} (${STATIC_AUDIO_ELEMENT_REASON})` - : `AI input waiting for ${STATIC_AUDIO_ELEMENT_REASON}`, true); - - const sendVideo = shouldSendVideoToAi(); - const shouldSendVideo = sendVideo && isLive(selectedRemoteVideo) && !selectedRemoteVideo.muted; - if (shouldSendVideo) { - const videoTransceiver = aiPc.addTransceiver(selectedRemoteVideo, { direction: 'sendonly' }); - aiVideoSender = videoTransceiver.sender; - log(`AI video starts with ${trackLabel(selectedRemoteVideo)} (${STATIC_VIDEO_ELEMENT_REASON})`); - } else { - aiVideoSender = null; - log(sendVideo - ? `AI video disabled: waiting for live track at ${STATIC_VIDEO_ELEMENT_REASON}` - : 'AI video uplink disabled by panel setting'); - } - - aiPc.ontrack = (event) => { - if (event.track.kind !== 'audio') return; - aiOutputTrackIds.add(event.track.id); - remoteAudio.delete(event.track.id); - aiAudioTrack = event.track; - log(`received AI audio ${trackLabel(aiAudioTrack)}`, true); - aiAudioTrack.onunmute = () => { - log(`AI audio unmuted ${trackLabel(aiAudioTrack)}`); - attachAiOutputToBusiness(); - }; - attachAiOutputToBusiness(); - }; - - aiPc.onicecandidate = (event) => { - if (aiWs?.readyState !== WebSocket.OPEN) return; - aiWs.send(JSON.stringify({ - type: 'ice-candidate', - payload: { - pc_id: id, - candidate: event.candidate ? { - candidate: event.candidate.candidate, - sdpMid: event.candidate.sdpMid, - sdpMLineIndex: event.candidate.sdpMLineIndex, - } : null, - }, - })); - }; - - setStatus('connecting'); - aiWs = new WebSocket(signalingUrl); - aiWs.onopen = async () => { - const offer = await aiPc.createOffer(); - await aiPc.setLocalDescription(offer); - aiWs.send(JSON.stringify({ - type: 'offer', - payload: { - pc_id: id, - sdp: aiPc.localDescription.sdp, - type: aiPc.localDescription.type, - assistant_id: assistantId, - vision_enabled: shouldSendVideo, - }, - })); - log('sent AI WebRTC offer', true); - }; - aiWs.onmessage = async (event) => { - try { - const msg = JSON.parse(event.data); - if (msg.type === 'answer') { - await aiPc.setRemoteDescription({ type: 'answer', sdp: msg.payload.sdp }); - log(`AI WebRTC answer applied; ${aiState()}`, true); - } else if (msg.type === 'ice-candidate' && msg.payload?.candidate) { - await aiPc.addIceCandidate(msg.payload.candidate).catch(() => {}); - } else if (msg.type === 'error') { - throw new Error(msg.payload?.message || 'server error'); - } - } catch (error) { - log(`signaling error: ${error.message || error}`); - } - }; - aiWs.onerror = () => { - setStatus('failed'); - log('AI signaling websocket error'); - }; - aiWs.onclose = () => setStatus('disconnected'); - - attachAudioToAi(); - attachVideoToAi(); - } - - function disconnectAi() { - try { aiDc?.close(); } catch {} - try { aiPc?.close(); } catch {} - try { aiWs?.close(); } catch {} - aiDc = null; - aiPc = null; - aiWs = null; - aiAudioSender = null; - aiVideoSender = null; - aiAudioTrack = null; - if (inputRecorder?.state === 'recording') stopInputRecording(); - setStatus('disconnected'); - } - - function setStatus(status) { - const el = document.getElementById('element-copilot-status'); - if (!el) return; - el.textContent = { - disconnected: '未连接', - connecting: '连接中', - connected: '已连接', - failed: '连接失败', - }[status] || status; - } - - function updateModeButton() { - const button = document.getElementById('element-copilot-toggle'); - if (!button) return; - if (isAiTakeover) { - button.textContent = '当前：AI 接管｜点击切回人工'; - button.style.background = '#8b5cf6'; - } else { - button.textContent = '当前：人工接管｜点击切换 AI'; - button.style.background = '#10b981'; - } - } - - window.__aiNativeElementDump = function () { - return { - staticAudioElement: STATIC_AUDIO_ELEMENT_REASON, - staticVideoElement: STATIC_VIDEO_ELEMENT_REASON, - selectedInput: trackLabel(selectedRemoteAudio), - selectedVideo: trackLabel(selectedRemoteVideo), - aiVideoSenderTrack: trackLabel(aiVideoSender?.track), - selectedOutput: trackLabel(selectedBusinessSender?.track), - originalOutput: trackLabel(originalBusinessTrack), - aiAudio: trackLabel(aiAudioTrack), - aiAttached, - aiState: aiState(), - recording: inputRecorder?.state === 'recording' ? { - track: trackLabel(inputRecordingTrack), - seconds: ((Date.now() - inputRecordingStartedAt) / 1000).toFixed(1), - chunks: inputRecordingChunks.length, - } : null, - inputs: Array.from(remoteAudio.values()).map((item) => ({ - selected: item.track === selectedRemoteAudio, - id: item.track.id, - muted: item.track.muted, - live: item.track.readyState, - hasReceiver: Boolean(item.receiver), - matchesStaticElement: hasElementReason(item, STATIC_AUDIO_ELEMENT_REASON), - bytesReceived: item.bytesReceived || 0, - packetsReceived: item.packetsReceived || 0, - bytesDelta: item.bytesDelta, - packetsDelta: item.packetsDelta, - reasons: Array.from(item.reasons), - })), - videos: Array.from(remoteVideo.values()).map((item) => ({ - selected: item.track === selectedRemoteVideo, - id: item.track.id, - muted: item.track.muted, - live: item.track.readyState, - hasReceiver: Boolean(item.receiver), - matchesStaticElement: hasElementReason(item, STATIC_VIDEO_ELEMENT_REASON), - reasons: Array.from(item.reasons), - })), - outputs: Array.from(businessSenders.values()).map((item) => ({ - selected: item.sender === selectedBusinessSender, - id: item.id, - track: trackLabel(item.sender.track || item.track), - originalTrack: trackLabel(item.originalTrack), - pcAddTrack: item.sources.has('pc-addTrack'), - isRemoteInput: isRemoteInputTrack(item.sender.track || item.track), - bytesSent: item.bytesSent, - packetsSent: item.packetsSent, - bytesDelta: item.bytesDelta, - packetsDelta: item.packetsDelta, - sources: Array.from(item.sources), - })), - }; - }; - - function renderPanel() { - if (!document.body || document.getElementById('element-copilot-panel')) return; - const panel = document.createElement('div'); - panel.id = 'element-copilot-panel'; - panel.style.cssText = [ - 'position:fixed', - 'top:20px', - 'left:20px', - 'width:360px', - 'z-index:999999', - 'box-sizing:border-box', - 'padding:16px', - 'border-radius:10px', - 'background:rgba(17,24,39,.96)', - 'color:white', - 'font-family:system-ui,-apple-system,BlinkMacSystemFont,sans-serif', - 'box-shadow:0 12px 32px rgba(0,0,0,.35)', - ].join(';'); - panel.innerHTML = ` -
- AI 元素选轨 - 未连接 -
-
- 音频输入：${STATIC_AUDIO_ELEMENT_REASON}
- 视频输入：${STATIC_VIDEO_ELEMENT_REASON} -
- - - - -
- - -
-
-
${STATIC_AUDIO_ELEMENT_REASON}原生音轨
-
-
-
-
AI 输出业务上行
-
-
- - -
- 等待交互... -
- `; - document.body.appendChild(panel); - - const toggle = document.getElementById('element-copilot-toggle'); - const recordButton = document.getElementById('element-copilot-record'); - const urlInput = document.getElementById('element-copilot-url'); - const assistantInput = document.getElementById('element-copilot-assistant'); - const sendAudioInput = document.getElementById('element-copilot-send-audio'); - const sendVideoInput = document.getElementById('element-copilot-send-video'); - sendAudioInput.checked = shouldSendAudioToAi(); - sendVideoInput.checked = shouldSendVideoToAi(); - sendAudioInput.addEventListener('change', () => { - writeBoolSetting(AUDIO_UPLINK_STORAGE_KEY, sendAudioInput.checked); - log(`AI audio uplink setting: ${sendAudioInput.checked ? 'on' : 'off'}`); - }); - sendVideoInput.addEventListener('change', () => { - writeBoolSetting(VIDEO_UPLINK_STORAGE_KEY, sendVideoInput.checked); - log(`AI video uplink setting: ${sendVideoInput.checked ? 'on' : 'off'}`); - }); - recordButton.addEventListener('click', toggleInputRecording); - toggle.addEventListener('click', async () => { - if (!isAiTakeover && !assistantInput.value.trim()) { - alert('请先填写助手 ID。'); - return; - } - isAiTakeover = !isAiTakeover; - updateModeButton(); - if (isAiTakeover) { - await connectAi(urlInput.value.trim(), assistantInput.value.trim()); - await attachAiOutputToBusiness(); - } else { - await restoreBusinessAudio(); - disconnectAi(); - } - }); - updateModeButton(); - updateRecordButton(); - } - - function renderPanelSoon() { - try { renderPanel(); } catch (error) { console.warn('[AI Native Element] panel error', error); } - } - - if (document.readyState === 'loading') { - document.addEventListener('DOMContentLoaded', renderPanelSoon, { once: true }); - } else { - renderPanelSoon(); - } - requestAnimationFrame(renderVu); - setInterval(refreshStats, 250); - setTimeout(renderPanelSoon, 1500); - setTimeout(renderPanelSoon, 3000); - window.addEventListener('beforeunload', () => { - if (inputRecorder?.state === 'recording') stopInputRecording(); - }); -})(); diff --git a/scripts/copilot-native-static.js b/scripts/copilot-native-static.js deleted file mode 100644 index bbee563..0000000 --- a/scripts/copilot-native-static.js +++ /dev/null @@ -1,876 +0,0 @@ -// ==UserScript== -// @name 12345 AI Native Static Copilot -// @namespace http://tampermonkey.net/ -// @version 1.0 -// @description Native-track bridge with static input/output selection rules. -// @author You -// @match *://yhy-yw.22sj.cn:*/* -// @grant none -// @run-at document-start -// ==/UserScript== - -(function () { - 'use strict'; - - const DEFAULT_SIGNALING_URL = 'ws://182.92.86.220:8000/ws/voice'; - - const originalAddEventListener = RTCPeerConnection.prototype.addEventListener; - const originalAddTrack = RTCPeerConnection.prototype.addTrack; - const originalAddTransceiver = RTCPeerConnection.prototype.addTransceiver; - const originalSetRemoteDescription = RTCPeerConnection.prototype.setRemoteDescription; - const mediaSrcObjectDescriptor = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, 'srcObject'); - - const remoteAudio = new Map(); - const remoteVideo = new Map(); - const businessSenders = new Map(); - const aiOutputTrackIds = new Set(); - const inboundMemory = new Map(); - const outboundMemory = new Map(); - const internalPcs = new WeakSet(); - const ENABLE_VIDEO_UPLINK = true; - - let selectedRemoteAudio = null; - let selectedRemoteVideo = null; - let selectedBusinessSender = null; - let originalBusinessTrack = null; - let aiAudioTrack = null; - - let aiPc = null; - let aiWs = null; - let aiDc = null; - let aiAudioSender = null; - let aiVideoSender = null; - let isAiTakeover = false; - let aiAttached = false; - let inputVu = 0; - let outputVu = 0; - let inputHealthTimer = null; - let outputHealthTimer = null; - let assistantLine = null; - let assistantBuffer = ''; - let inputRecorder = null; - let inputRecordingChunks = []; - let inputRecordingTrack = null; - let inputRecordingStartedAt = 0; - - function log(message) { - console.log(`[AI Native Static] ${message}`); - } - - function appendChatLine(role, text, important = false) { - const box = document.getElementById('static-copilot-log'); - if (!box) return; - if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = ''; - const row = document.createElement('div'); - row.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] ${role}: ${text}`; - row.style.marginBottom = '4px'; - if (important) row.style.color = '#fcd34d'; - box.appendChild(row); - while (box.children.length > 120) box.removeChild(box.firstChild); - box.scrollTop = box.scrollHeight; - return row; - } - - function startAssistantLine() { - assistantBuffer = ''; - assistantLine = null; - } - - function appendAssistantDelta(delta) { - if (!delta) return; - const box = document.getElementById('static-copilot-log'); - if (!box) { - console.log(`[AI Native Static] AI: ${delta}`); - return; - } - if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = ''; - if (!assistantLine) assistantLine = appendChatLine('AI', ''); - assistantBuffer += delta; - assistantLine.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] AI: ${assistantBuffer}`; - box.scrollTop = box.scrollHeight; - } - - function finishAssistantLine() { - assistantLine = null; - assistantBuffer = ''; - } - - function logUserTranscript(text) { - if (!text?.trim()) return; - finishAssistantLine(); - appendChatLine('用户', text.trim(), true); - } - - function recorderMimeType() { - if (window.MediaRecorder?.isTypeSupported?.('audio/webm;codecs=opus')) return 'audio/webm;codecs=opus'; - if (window.MediaRecorder?.isTypeSupported?.('audio/webm')) return 'audio/webm'; - return ''; - } - - function recordingFileName(track) { - const stamp = new Date().toISOString().replace(/[:.]/g, '-'); - const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track'; - return `ai-input-${trackPart}-${stamp}.webm`; - } - - async function startInputRecording() { - if (inputRecorder?.state === 'recording') return; - selectRemoteAudio('record'); - if (!isLive(selectedRemoteAudio)) { - appendChatLine('录音', '没有可录制的远端输入音轨', true); - return; - } - if (!window.MediaRecorder) { - appendChatLine('录音', '当前浏览器不支持 MediaRecorder', true); - return; - } - - inputRecordingChunks = []; - inputRecordingTrack = selectedRemoteAudio; - inputRecordingStartedAt = Date.now(); - const stream = new MediaStream([selectedRemoteAudio]); - const mimeType = recorderMimeType(); - inputRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); - inputRecorder.ondataavailable = (event) => { - if (event.data?.size > 0) inputRecordingChunks.push(event.data); - }; - inputRecorder.onstop = () => { - const blob = new Blob(inputRecordingChunks, { type: inputRecorder?.mimeType || 'audio/webm' }); - const url = URL.createObjectURL(blob); - const link = document.createElement('a'); - link.href = url; - link.download = recordingFileName(inputRecordingTrack); - document.body.appendChild(link); - link.click(); - link.remove(); - setTimeout(() => URL.revokeObjectURL(url), 30000); - - const seconds = ((Date.now() - inputRecordingStartedAt) / 1000).toFixed(1); - appendChatLine('录音', `已保存 ${link.download}（${seconds}s，${(blob.size / 1024).toFixed(1)} KB）`, true); - inputRecorder = null; - inputRecordingChunks = []; - inputRecordingTrack = null; - updateRecordButton(); - }; - inputRecorder.start(250); - appendChatLine('录音', `开始录制 AI 输入：${trackLabel(inputRecordingTrack)}`, true); - updateRecordButton(); - } - - function stopInputRecording() { - if (inputRecorder?.state === 'recording') inputRecorder.stop(); - } - - function toggleInputRecording() { - if (inputRecorder?.state === 'recording') stopInputRecording(); - else startInputRecording(); - } - - function updateRecordButton() { - const button = document.getElementById('static-copilot-record'); - if (!button) return; - const recording = inputRecorder?.state === 'recording'; - button.textContent = recording ? '停止录制 AI 输入' : '录制 AI 输入'; - button.style.background = recording ? '#dc2626' : '#2563eb'; - } - - function pcId() { - const bytes = new Uint8Array(16); - crypto.getRandomValues(bytes); - return 'PC-' + Array.from(bytes).map((b) => b.toString(16).padStart(2, '0')).join(''); - } - - function isInternalPc(pc) { - return Boolean(pc && (pc.__aiNativeStaticInternal || internalPcs.has(pc) || pc === aiPc)); - } - - function isLive(track) { - return track && track.readyState === 'live'; - } - - function trackLabel(track) { - if (!track) return 'none'; - return `${track.id}${track.muted ? ' muted' : ''}`; - } - - function markInternalPc(pc) { - if (!pc) return; - pc.__aiNativeStaticInternal = true; - internalPcs.add(pc); - } - - function rememberRemoteReceiver(receiver, reason) { - const track = receiver?.track; - if (!isLive(track) || isInternalPc(receiver?.transport)) return; - - if (track.kind === 'audio' && !aiOutputTrackIds.has(track.id)) { - let item = remoteAudio.get(track.id); - if (!item) { - item = { track, receiver, reasons: new Set(), bytesDelta: 0, packetsDelta: 0 }; - remoteAudio.set(track.id, item); - log(`audio candidate ${trackLabel(track)} (${reason})`, true); - } - item.track = track; - item.receiver = receiver || item.receiver; - item.reasons.add(reason); - selectRemoteAudio(reason); - } - - if (track.kind === 'video') { - let item = remoteVideo.get(track.id); - if (!item) { - item = { track, receiver, reasons: new Set() }; - remoteVideo.set(track.id, item); - log(`video candidate ${trackLabel(track)} (${reason})`, true); - } - item.track = track; - item.receiver = receiver || item.receiver; - item.reasons.add(reason); - if (!selectedRemoteVideo || selectedRemoteVideo.readyState !== 'live') { - selectedRemoteVideo = track; - attachVideoToAi(); - } - } - } - - function rememberRemoteStream(stream, reason) { - if (!stream || !(stream instanceof MediaStream)) return; - for (const track of stream.getAudioTracks()) { - if (aiOutputTrackIds.has(track.id)) continue; - let item = remoteAudio.get(track.id); - if (!item) { - item = { track, receiver: null, reasons: new Set(), bytesDelta: 0, packetsDelta: 0 }; - remoteAudio.set(track.id, item); - log(`audio stream candidate ${trackLabel(track)} (${reason})`, true); - } - item.track = track; - item.reasons.add(reason); - } - for (const track of stream.getVideoTracks()) { - let item = remoteVideo.get(track.id); - if (!item) { - item = { track, receiver: null, reasons: new Set() }; - remoteVideo.set(track.id, item); - log(`video stream candidate ${trackLabel(track)} (${reason})`, true); - } - item.track = track; - item.reasons.add(reason); - if (!selectedRemoteVideo || selectedRemoteVideo.readyState !== 'live') selectedRemoteVideo = track; - } - selectRemoteAudio(reason); - attachVideoToAi(); - } - - function selectRemoteAudio(reason) { - if (isLive(selectedRemoteAudio) && !selectedRemoteAudio.muted) return selectedRemoteAudio; - - const item = Array.from(remoteAudio.values()) - .find((candidate) => isLive(candidate.track) && !candidate.track.muted && candidate.receiver); - if (!item) return null; - - selectedRemoteAudio = item.track; - log(`selected input audio (${reason}): ${trackLabel(selectedRemoteAudio)}`, true); - attachAudioToAi(); - return selectedRemoteAudio; - } - - function rememberBusinessSender(sender, track, reason) { - if (!sender) return; - const senderTrack = track || sender.track; - if (senderTrack && senderTrack.kind !== 'audio') return; - - const id = sender.__aiNativeStaticSenderId || `sender-${businessSenders.size + 1}`; - sender.__aiNativeStaticSenderId = id; - - let item = businessSenders.get(id); - if (!item) { - item = { id, sender, track: senderTrack || null, originalTrack: senderTrack || null, sources: new Set(), bytesDelta: 0, packetsDelta: 0 }; - businessSenders.set(id, item); - log(`business sender candidate ${id} (${reason}): ${trackLabel(senderTrack)}`, true); - } - item.sender = sender; - item.track = sender.track || senderTrack || item.track; - if (!isAiTakeover && senderTrack && senderTrack !== aiAudioTrack) item.originalTrack = senderTrack; - item.sources.add(reason); - wrapBusinessSender(sender); - selectBusinessSender(reason); - } - - function wrapBusinessSender(sender) { - if (!sender || sender.__aiNativeStaticWrapped) return; - const originalReplaceTrack = sender.replaceTrack?.bind(sender); - if (!originalReplaceTrack) return; - - sender.replaceTrack = async function (track) { - const result = await originalReplaceTrack(track); - rememberBusinessSender(sender, track, 'replaceTrack'); - return result; - }; - sender.__aiNativeStaticWrapped = true; - } - - function isRemoteInputTrack(track) { - return Boolean(track && remoteAudio.has(track.id) && !aiOutputTrackIds.has(track.id)); - } - - function selectBusinessSender(reason) { - if (selectedBusinessSender && !isRemoteInputTrack(selectedBusinessSender.track)) return selectedBusinessSender; - - const item = Array.from(businessSenders.values()).find((candidate) => { - const track = candidate.sender.track || candidate.track || candidate.originalTrack; - return candidate.sources.has('pc-addTrack') && !isRemoteInputTrack(track); - }); - if (!item) return null; - - selectedBusinessSender = item.sender; - originalBusinessTrack = item.originalTrack || item.sender.track || originalBusinessTrack; - log(`selected output sender (${reason}): ${item.id}, track=${trackLabel(item.sender.track)}`, true); - attachAiOutputToBusiness(); - return selectedBusinessSender; - } - - function setVu(id, level) { - const el = document.getElementById(id); - if (el) el.style.width = `${Math.min(100, level * 2500)}%`; - } - - function renderVu() { - inputVu *= 0.88; - outputVu *= 0.88; - setVu('static-copilot-input-vu', inputVu); - setVu('static-copilot-output-vu', outputVu); - requestAnimationFrame(renderVu); - } - - async function refreshStats() { - await refreshInputStats(); - await refreshOutputStats(); - } - - async function refreshInputStats() { - for (const item of remoteAudio.values()) { - if (!item.receiver?.getStats) continue; - const report = await item.receiver.getStats(); - report.forEach((stat) => { - const kind = stat.kind || stat.mediaType; - if (stat.type !== 'inbound-rtp' || kind !== 'audio') return; - const prev = inboundMemory.get(stat.id); - const bytes = stat.bytesReceived || 0; - const packets = stat.packetsReceived || 0; - item.bytesReceived = bytes; - item.packetsReceived = packets; - item.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0; - item.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0; - inboundMemory.set(stat.id, { bytes, packets }); - if (item.track === selectedRemoteAudio) { - inputVu = Math.max(inputVu, Math.min(0.12, item.bytesDelta / 12000), Math.min(0.04, item.packetsDelta / 500)); - } - }); - } - } - - async function refreshOutputStats() { - for (const item of businessSenders.values()) { - if (!item.sender?.getStats) continue; - item.track = item.sender.track || item.track; - const report = await item.sender.getStats(); - report.forEach((stat) => { - const kind = stat.kind || stat.mediaType; - if (stat.type !== 'outbound-rtp' || kind !== 'audio') return; - const key = `${item.id}:${stat.id}`; - const prev = outboundMemory.get(key); - const bytes = stat.bytesSent || 0; - const packets = stat.packetsSent || 0; - item.bytesSent = bytes; - item.packetsSent = packets; - item.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0; - item.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0; - outboundMemory.set(key, { bytes, packets }); - if (item.sender === selectedBusinessSender) { - outputVu = Math.max(outputVu, Math.min(0.12, item.bytesDelta / 12000), Math.min(0.04, item.packetsDelta / 500)); - } - }); - } - } - - async function outboundBytes(sender) { - if (!sender?.getStats) return null; - let total = 0; - const report = await sender.getStats(); - report.forEach((stat) => { - const kind = stat.kind || stat.mediaType; - if (stat.type === 'outbound-rtp' && kind === 'audio') total += stat.bytesSent || 0; - }); - return total; - } - - function aiState() { - if (!aiPc) return 'pc=null'; - return `conn=${aiPc.connectionState},ice=${aiPc.iceConnectionState},signaling=${aiPc.signalingState},remote=${aiPc.remoteDescription?.type || 'none'}`; - } - - function aiReady() { - return Boolean(aiPc?.remoteDescription && ( - aiPc.connectionState === 'connected' - || aiPc.iceConnectionState === 'connected' - || aiPc.iceConnectionState === 'completed' - )); - } - - function waitForAiReady(timeoutMs = 10000) { - if (aiReady()) return Promise.resolve(true); - return new Promise((resolve) => { - const startedAt = Date.now(); - const timer = setInterval(() => { - if (aiReady()) { - clearInterval(timer); - resolve(true); - } else if (Date.now() - startedAt >= timeoutMs) { - clearInterval(timer); - resolve(false); - } - }, 250); - }); - } - - function checkAiInput(track) { - if (inputHealthTimer) clearTimeout(inputHealthTimer); - inputHealthTimer = setTimeout(async () => { - if (!aiAudioSender || aiAudioSender.track !== track) return; - await waitForAiReady(); - const before = await outboundBytes(aiAudioSender); - await new Promise((resolve) => setTimeout(resolve, 2500)); - const after = await outboundBytes(aiAudioSender); - log(`AI input packet check: before=${before}, after=${after}, ${aiState()}`, true); - }, 500); - } - - function checkAiOutput(track) { - if (outputHealthTimer) clearTimeout(outputHealthTimer); - outputHealthTimer = setTimeout(async () => { - if (!selectedBusinessSender || selectedBusinessSender.track !== track) return; - const before = await outboundBytes(selectedBusinessSender); - await new Promise((resolve) => setTimeout(resolve, 2500)); - const after = await outboundBytes(selectedBusinessSender); - log(`AI output packet check: before=${before}, after=${after}, sender=${trackLabel(selectedBusinessSender.track)}`, true); - }, 800); - } - - async function attachAudioToAi() { - if (!aiAudioSender || !isLive(selectedRemoteAudio)) return false; - if (aiAudioSender.track === selectedRemoteAudio) return true; - await aiAudioSender.replaceTrack(selectedRemoteAudio); - log(`AI input uses ${trackLabel(selectedRemoteAudio)}`, true); - checkAiInput(selectedRemoteAudio); - return true; - } - - async function attachVideoToAi() { - if (!aiVideoSender || !isLive(selectedRemoteVideo)) return false; - if (aiVideoSender.track === selectedRemoteVideo) return true; - await aiVideoSender.replaceTrack(selectedRemoteVideo); - log(`AI video uses ${trackLabel(selectedRemoteVideo)}`, true); - return true; - } - - async function attachAiOutputToBusiness() { - selectBusinessSender('before-output'); - if (!isAiTakeover || !selectedBusinessSender || !isLive(aiAudioTrack)) { - log(`AI output waiting: takeover=${isAiTakeover}, sender=${Boolean(selectedBusinessSender)}, aiTrack=${trackLabel(aiAudioTrack)}`); - return false; - } - if (selectedBusinessSender.track === aiAudioTrack) return true; - await selectedBusinessSender.replaceTrack(aiAudioTrack); - aiAttached = true; - log(`business sender uses AI audio ${trackLabel(aiAudioTrack)}`, true); - checkAiOutput(aiAudioTrack); - return true; - } - - async function restoreBusinessAudio() { - if (!selectedBusinessSender || !aiAttached) return; - await selectedBusinessSender.replaceTrack(originalBusinessTrack || null); - aiAttached = false; - log(`business sender restored ${trackLabel(originalBusinessTrack)}`, true); - } - - function wrapTrackListener(listener) { - if (!listener || listener.__aiNativeStaticWrapped) return listener; - const wrapped = function (event) { - const ownerPc = this instanceof RTCPeerConnection ? this : event?.currentTarget; - if (!isInternalPc(ownerPc) && event?.track) { - rememberRemoteReceiver(event.receiver, 'pc-track'); - for (const stream of event.streams || []) rememberRemoteStream(stream, 'pc-track-stream'); - } - if (typeof listener === 'function') return listener.apply(this, arguments); - if (listener?.handleEvent) return listener.handleEvent(event); - return undefined; - }; - wrapped.__aiNativeStaticWrapped = true; - return wrapped; - } - - RTCPeerConnection.prototype.addEventListener = function (type, listener, options) { - if (type === 'track') return originalAddEventListener.call(this, type, wrapTrackListener(listener), options); - return originalAddEventListener.call(this, type, listener, options); - }; - - try { - const descriptor = Object.getOwnPropertyDescriptor(RTCPeerConnection.prototype, 'ontrack'); - Object.defineProperty(RTCPeerConnection.prototype, 'ontrack', { - configurable: true, - enumerable: true, - get() { - return descriptor?.get ? descriptor.get.call(this) : this.__aiNativeStaticOnTrack || null; - }, - set(listener) { - const wrapped = wrapTrackListener(listener); - this.__aiNativeStaticOnTrack = wrapped; - if (descriptor?.set) descriptor.set.call(this, wrapped); - else this.addEventListener('track', wrapped); - }, - }); - } catch (error) { - console.warn('[AI Native Static] failed to wrap ontrack', error); - } - - RTCPeerConnection.prototype.addTrack = function (track, ...streams) { - const sender = originalAddTrack.call(this, track, ...streams); - if (!isInternalPc(this) && track?.kind === 'audio') rememberBusinessSender(sender, track, 'pc-addTrack'); - return sender; - }; - - if (originalAddTransceiver) { - RTCPeerConnection.prototype.addTransceiver = function (trackOrKind, init) { - const transceiver = originalAddTransceiver.call(this, trackOrKind, init); - if (!isInternalPc(this)) { - const kind = trackOrKind instanceof MediaStreamTrack ? trackOrKind.kind : trackOrKind; - if (kind === 'audio') { - const track = trackOrKind instanceof MediaStreamTrack ? trackOrKind : transceiver.sender?.track; - rememberBusinessSender(transceiver.sender, track, 'pc-addTransceiver'); - } - } - return transceiver; - }; - } - - RTCPeerConnection.prototype.setRemoteDescription = function () { - const result = originalSetRemoteDescription.apply(this, arguments); - if (!isInternalPc(this)) { - Promise.resolve(result).then(() => { - for (const receiver of this.getReceivers?.() || []) rememberRemoteReceiver(receiver, 'receiver-after-srd'); - for (const sender of this.getSenders?.() || []) rememberBusinessSender(sender, sender.track, 'sender-after-srd'); - }).catch(() => {}); - } - return result; - }; - - if (mediaSrcObjectDescriptor?.set) { - Object.defineProperty(HTMLMediaElement.prototype, 'srcObject', { - configurable: true, - enumerable: mediaSrcObjectDescriptor.enumerable, - get() { - return mediaSrcObjectDescriptor.get.call(this); - }, - set(stream) { - if (!this.__aiNativeStaticElement) rememberRemoteStream(stream, `${this.tagName.toLowerCase()}-srcObject`); - return mediaSrcObjectDescriptor.set.call(this, stream); - }, - }); - } - - setInterval(() => { - for (const element of document.querySelectorAll('audio, video')) { - if (!element.__aiNativeStaticElement) rememberRemoteStream(element.srcObject, 'media-scan'); - } - }, 1000); - - async function connectAi(signalingUrl, assistantId) { - disconnectAi(); - - const id = pcId(); - aiPc = new RTCPeerConnection({ iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] }); - markInternalPc(aiPc); - aiPc.onconnectionstatechange = () => log(`AI PC connectionState=${aiPc.connectionState}`); - aiPc.oniceconnectionstatechange = () => log(`AI PC iceConnectionState=${aiPc.iceConnectionState}`); - - aiDc = aiPc.createDataChannel('chat'); - aiDc.onopen = () => { - setStatus('connected'); - log('AI data channel open', true); - aiDc.send(JSON.stringify({ type: 'client-ready' })); - }; - aiDc.onclose = () => setStatus('disconnected'); - aiDc.onmessage = (event) => { - try { - const msg = JSON.parse(event.data); - const content = msg.delta || msg.content; - if (msg.type === 'transcript') { - logUserTranscript(content); - } else if (msg.type === 'assistant-text-start') { - startAssistantLine(); - } else if (msg.type === 'assistant-text-delta') { - appendAssistantDelta(content); - } else if (msg.type === 'assistant-text-end') { - finishAssistantLine(); - } else { - log(`data channel message: ${msg.type}`); - } - } catch { - appendChatLine('DATA', event.data); - } - }; - - selectRemoteAudio('connect'); - selectBusinessSender('connect'); - - const audioTransceiver = isLive(selectedRemoteAudio) - ? aiPc.addTransceiver(selectedRemoteAudio, { direction: 'sendrecv' }) - : aiPc.addTransceiver('audio', { direction: 'sendrecv' }); - aiAudioSender = audioTransceiver.sender; - log(isLive(selectedRemoteAudio) ? `AI input starts with ${trackLabel(selectedRemoteAudio)}` : 'AI input waiting for selected audio', true); - - const shouldSendVideo = ENABLE_VIDEO_UPLINK && isLive(selectedRemoteVideo) && !selectedRemoteVideo.muted; - if (shouldSendVideo) { - const videoTransceiver = aiPc.addTransceiver(selectedRemoteVideo, { direction: 'sendonly' }); - aiVideoSender = videoTransceiver.sender; - log(`AI video starts with ${trackLabel(selectedRemoteVideo)}`); - } else { - aiVideoSender = null; - log('AI video disabled for this offer: no live native video track yet'); - } - - aiPc.ontrack = (event) => { - if (event.track.kind !== 'audio') return; - aiOutputTrackIds.add(event.track.id); - remoteAudio.delete(event.track.id); - aiAudioTrack = event.track; - log(`received AI audio ${trackLabel(aiAudioTrack)}`, true); - aiAudioTrack.onunmute = () => { - log(`AI audio unmuted ${trackLabel(aiAudioTrack)}`); - attachAiOutputToBusiness(); - }; - attachAiOutputToBusiness(); - }; - - aiPc.onicecandidate = (event) => { - if (aiWs?.readyState !== WebSocket.OPEN) return; - aiWs.send(JSON.stringify({ - type: 'ice-candidate', - payload: { - pc_id: id, - candidate: event.candidate ? { - candidate: event.candidate.candidate, - sdpMid: event.candidate.sdpMid, - sdpMLineIndex: event.candidate.sdpMLineIndex, - } : null, - }, - })); - }; - - setStatus('connecting'); - aiWs = new WebSocket(signalingUrl); - aiWs.onopen = async () => { - const offer = await aiPc.createOffer(); - await aiPc.setLocalDescription(offer); - aiWs.send(JSON.stringify({ - type: 'offer', - payload: { - pc_id: id, - sdp: aiPc.localDescription.sdp, - type: aiPc.localDescription.type, - assistant_id: assistantId, - vision_enabled: shouldSendVideo, - }, - })); - log('sent AI WebRTC offer', true); - }; - aiWs.onmessage = async (event) => { - try { - const msg = JSON.parse(event.data); - if (msg.type === 'answer') { - await aiPc.setRemoteDescription({ type: 'answer', sdp: msg.payload.sdp }); - log(`AI WebRTC answer applied; ${aiState()}`, true); - } else if (msg.type === 'ice-candidate' && msg.payload?.candidate) { - await aiPc.addIceCandidate(msg.payload.candidate).catch(() => {}); - } else if (msg.type === 'error') { - throw new Error(msg.payload?.message || 'server error'); - } - } catch (error) { - log(`signaling error: ${error.message || error}`); - } - }; - aiWs.onerror = () => { - setStatus('failed'); - log('AI signaling websocket error'); - }; - aiWs.onclose = () => setStatus('disconnected'); - - attachAudioToAi(); - attachVideoToAi(); - } - - function disconnectAi() { - try { aiDc?.close(); } catch {} - try { aiPc?.close(); } catch {} - try { aiWs?.close(); } catch {} - aiDc = null; - aiPc = null; - aiWs = null; - aiAudioSender = null; - aiVideoSender = null; - aiAudioTrack = null; - if (inputRecorder?.state === 'recording') stopInputRecording(); - setStatus('disconnected'); - } - - function setStatus(status) { - const el = document.getElementById('static-copilot-status'); - if (!el) return; - el.textContent = { - disconnected: '未连接', - connecting: '连接中', - connected: '已连接', - failed: '连接失败', - }[status] || status; - } - - function updateModeButton() { - const button = document.getElementById('static-copilot-toggle'); - if (!button) return; - if (isAiTakeover) { - button.textContent = '当前：AI 接管｜点击切回人工'; - button.style.background = '#8b5cf6'; - } else { - button.textContent = '当前：人工接管｜点击切换 AI'; - button.style.background = '#10b981'; - } - } - - window.__aiNativeStaticDump = function () { - return { - selectedInput: trackLabel(selectedRemoteAudio), - selectedVideo: trackLabel(selectedRemoteVideo), - aiVideoSenderTrack: trackLabel(aiVideoSender?.track), - selectedOutput: trackLabel(selectedBusinessSender?.track), - originalOutput: trackLabel(originalBusinessTrack), - aiAudio: trackLabel(aiAudioTrack), - aiAttached, - aiState: aiState(), - recording: inputRecorder?.state === 'recording' ? { - track: trackLabel(inputRecordingTrack), - seconds: ((Date.now() - inputRecordingStartedAt) / 1000).toFixed(1), - chunks: inputRecordingChunks.length, - } : null, - inputs: Array.from(remoteAudio.values()).map((item) => ({ - selected: item.track === selectedRemoteAudio, - id: item.track.id, - muted: item.track.muted, - live: item.track.readyState, - hasReceiver: Boolean(item.receiver), - bytesReceived: item.bytesReceived || 0, - packetsReceived: item.packetsReceived || 0, - bytesDelta: item.bytesDelta, - packetsDelta: item.packetsDelta, - reasons: Array.from(item.reasons), - })), - outputs: Array.from(businessSenders.values()).map((item) => ({ - selected: item.sender === selectedBusinessSender, - id: item.id, - track: trackLabel(item.sender.track || item.track), - originalTrack: trackLabel(item.originalTrack), - pcAddTrack: item.sources.has('pc-addTrack'), - isRemoteInput: isRemoteInputTrack(item.sender.track || item.track), - bytesSent: item.bytesSent, - packetsSent: item.packetsSent, - bytesDelta: item.bytesDelta, - packetsDelta: item.packetsDelta, - sources: Array.from(item.sources), - })), - }; - }; - - function renderPanel() { - if (!document.body || document.getElementById('static-copilot-panel')) return; - const panel = document.createElement('div'); - panel.id = 'static-copilot-panel'; - panel.style.cssText = [ - 'position:fixed', - 'top:20px', - 'left:20px', - 'width:340px', - 'z-index:999999', - 'box-sizing:border-box', - 'padding:16px', - 'border-radius:10px', - 'background:rgba(17,24,39,.96)', - 'color:white', - 'font-family:system-ui,-apple-system,BlinkMacSystemFont,sans-serif', - 'box-shadow:0 12px 32px rgba(0,0,0,.35)', - ].join(';'); - panel.innerHTML = ` -
- AI 原生转接 - 未连接 -
- - - - -
-
远端输入原生音轨
-
-
-
-
AI 输出业务上行
-
-
- - -
- 等待交互... -
- `; - document.body.appendChild(panel); - - const toggle = document.getElementById('static-copilot-toggle'); - const recordButton = document.getElementById('static-copilot-record'); - const urlInput = document.getElementById('static-copilot-url'); - const assistantInput = document.getElementById('static-copilot-assistant'); - recordButton.addEventListener('click', toggleInputRecording); - toggle.addEventListener('click', async () => { - if (!isAiTakeover && !assistantInput.value.trim()) { - alert('请先填写助手 ID。'); - return; - } - isAiTakeover = !isAiTakeover; - updateModeButton(); - if (isAiTakeover) { - await connectAi(urlInput.value.trim(), assistantInput.value.trim()); - await attachAiOutputToBusiness(); - } else { - await restoreBusinessAudio(); - disconnectAi(); - } - }); - updateModeButton(); - updateRecordButton(); - } - - function renderPanelSoon() { - try { renderPanel(); } catch (error) { console.warn('[AI Native Static] panel error', error); } - } - - if (document.readyState === 'loading') { - document.addEventListener('DOMContentLoaded', renderPanelSoon, { once: true }); - } else { - renderPanelSoon(); - } - requestAnimationFrame(renderVu); - setInterval(refreshStats, 250); - setTimeout(renderPanelSoon, 1500); - setTimeout(renderPanelSoon, 3000); - window.addEventListener('beforeunload', () => { - if (inputRecorder?.state === 'recording') stopInputRecording(); - }); -})();