// ==UserScript== // @name 12345 AI Native Element Copilot // @namespace http://tampermonkey.net/ // @version 1.0 // @description Native-track bridge with fixed element selection: element-2:audio input, element-0:video vision. // @author You // @match *://yhy-yw.22sj.cn:*/* // @grant none // @run-at document-start // ==/UserScript== (function () { 'use strict'; const DEFAULT_SIGNALING_URL = 'ws://182.92.86.220:8000/ws/voice'; const STATIC_AUDIO_ELEMENT_REASON = 'element-2:audio'; const STATIC_VIDEO_ELEMENT_REASON = 'element-0:video'; const originalAddEventListener = RTCPeerConnection.prototype.addEventListener; const originalAddTrack = RTCPeerConnection.prototype.addTrack; const originalAddTransceiver = RTCPeerConnection.prototype.addTransceiver; const originalSetRemoteDescription = RTCPeerConnection.prototype.setRemoteDescription; const mediaSrcObjectDescriptor = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, 'srcObject'); const remoteAudio = new Map(); const remoteVideo = new Map(); const businessSenders = new Map(); const aiOutputTrackIds = new Set(); const inboundMemory = new Map(); const outboundMemory = new Map(); const internalPcs = new WeakSet(); const AUDIO_UPLINK_STORAGE_KEY = 'element-copilot-send-audio'; const VIDEO_UPLINK_STORAGE_KEY = 'element-copilot-send-video'; let selectedRemoteAudio = null; let selectedRemoteVideo = null; let selectedBusinessSender = null; let originalBusinessTrack = null; let aiAudioTrack = null; let aiPc = null; let aiWs = null; let aiDc = null; let aiAudioSender = null; let aiVideoSender = null; let isAiTakeover = false; let aiAttached = false; let inputVu = 0; let outputVu = 0; let inputHealthTimer = null; let outputHealthTimer = null; let assistantLine = null; let assistantBuffer = ''; let inputRecorder = null; let inputRecordingChunks = []; let inputRecordingTrack = null; let inputRecordingStartedAt = 0; function log(message) { console.log(`[AI Native Element] ${message}`); } function readBoolSetting(key, fallback) { try { const value = localStorage.getItem(key); if (value === null) return fallback; return value === 'true'; } catch { return fallback; } } function writeBoolSetting(key, value) { try { localStorage.setItem(key, value ? 'true' : 'false'); } catch { // Ignore storage failures; the current checkbox state still applies. } } function shouldSendAudioToAi() { return readBoolSetting(AUDIO_UPLINK_STORAGE_KEY, true); } function shouldSendVideoToAi() { return readBoolSetting(VIDEO_UPLINK_STORAGE_KEY, true); } function appendChatLine(role, text, important = false) { const box = document.getElementById('element-copilot-log'); if (!box) return; if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = ''; const row = document.createElement('div'); row.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] ${role}: ${text}`; row.style.marginBottom = '4px'; if (important) row.style.color = '#fcd34d'; box.appendChild(row); while (box.children.length > 120) box.removeChild(box.firstChild); box.scrollTop = box.scrollHeight; return row; } function startAssistantLine() { assistantBuffer = ''; assistantLine = null; } function appendAssistantDelta(delta) { if (!delta) return; const box = document.getElementById('element-copilot-log'); if (!box) { console.log(`[AI Native Element] AI: ${delta}`); return; } if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = ''; if (!assistantLine) assistantLine = appendChatLine('AI', ''); assistantBuffer += delta; assistantLine.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] AI: ${assistantBuffer}`; box.scrollTop = box.scrollHeight; } function finishAssistantLine() { assistantLine = null; assistantBuffer = ''; } function logUserTranscript(text) { if (!text?.trim()) return; finishAssistantLine(); appendChatLine('用户', text.trim(), true); } function recorderMimeType() { if (window.MediaRecorder?.isTypeSupported?.('audio/webm;codecs=opus')) return 'audio/webm;codecs=opus'; if (window.MediaRecorder?.isTypeSupported?.('audio/webm')) return 'audio/webm'; return ''; } function recordingFileName(track) { const stamp = new Date().toISOString().replace(/[:.]/g, '-'); const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track'; return `ai-input-${trackPart}-${stamp}.webm`; } async function startInputRecording() { if (inputRecorder?.state === 'recording') return; selectRemoteAudio('record'); if (!isLive(selectedRemoteAudio)) { appendChatLine('录音', `没有可录制的 ${STATIC_AUDIO_ELEMENT_REASON} 音轨`, true); return; } if (!window.MediaRecorder) { appendChatLine('录音', '当前浏览器不支持 MediaRecorder', true); return; } inputRecordingChunks = []; inputRecordingTrack = selectedRemoteAudio; inputRecordingStartedAt = Date.now(); const stream = new MediaStream([selectedRemoteAudio]); const mimeType = recorderMimeType(); inputRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); inputRecorder.ondataavailable = (event) => { if (event.data?.size > 0) inputRecordingChunks.push(event.data); }; inputRecorder.onstop = () => { const blob = new Blob(inputRecordingChunks, { type: inputRecorder?.mimeType || 'audio/webm' }); const url = URL.createObjectURL(blob); const link = document.createElement('a'); link.href = url; link.download = recordingFileName(inputRecordingTrack); document.body.appendChild(link); link.click(); link.remove(); setTimeout(() => URL.revokeObjectURL(url), 30000); const seconds = ((Date.now() - inputRecordingStartedAt) / 1000).toFixed(1); appendChatLine('录音', `已保存 ${link.download}(${seconds}s,${(blob.size / 1024).toFixed(1)} KB)`, true); inputRecorder = null; inputRecordingChunks = []; inputRecordingTrack = null; updateRecordButton(); }; inputRecorder.start(250); appendChatLine('录音', `开始录制 AI 输入:${trackLabel(inputRecordingTrack)}`, true); updateRecordButton(); } function stopInputRecording() { if (inputRecorder?.state === 'recording') inputRecorder.stop(); } function toggleInputRecording() { if (inputRecorder?.state === 'recording') stopInputRecording(); else startInputRecording(); } function updateRecordButton() { const button = document.getElementById('element-copilot-record'); if (!button) return; const recording = inputRecorder?.state === 'recording'; button.textContent = recording ? '停止录制 AI 输入' : '录制 AI 输入'; button.style.background = recording ? '#dc2626' : '#2563eb'; } function pcId() { const bytes = new Uint8Array(16); crypto.getRandomValues(bytes); return 'PC-' + Array.from(bytes).map((b) => b.toString(16).padStart(2, '0')).join(''); } function isInternalPc(pc) { return Boolean(pc && (pc.__aiNativeElementInternal || internalPcs.has(pc) || pc === aiPc)); } function isLive(track) { return track && track.readyState === 'live'; } function trackLabel(track) { if (!track) return 'none'; return `${track.id}${track.muted ? ' muted' : ''}`; } function markInternalPc(pc) { if (!pc) return; pc.__aiNativeElementInternal = true; internalPcs.add(pc); } function elementReasonFor(element) { if (!element) return null; const tag = element.tagName?.toLowerCase(); if (tag !== 'audio' && tag !== 'video') return null; const elements = Array.from(document.querySelectorAll(tag)); const index = elements.indexOf(element); if (index < 0) return `${tag}-srcObject`; return `element-${index}:${tag}`; } function hasElementReason(item, reason) { return Boolean(item?.reasons?.has(reason)); } function rememberRemoteReceiver(receiver, reason) { const track = receiver?.track; if (!isLive(track) || isInternalPc(receiver?.transport)) return; if (track.kind === 'audio' && !aiOutputTrackIds.has(track.id)) { let item = remoteAudio.get(track.id); if (!item) { item = { track, receiver, reasons: new Set(), bytesDelta: 0, packetsDelta: 0 }; remoteAudio.set(track.id, item); log(`audio candidate ${trackLabel(track)} (${reason})`, true); } item.track = track; item.receiver = receiver || item.receiver; item.reasons.add(reason); selectRemoteAudio(reason); } if (track.kind === 'video') { let item = remoteVideo.get(track.id); if (!item) { item = { track, receiver, reasons: new Set() }; remoteVideo.set(track.id, item); log(`video candidate ${trackLabel(track)} (${reason})`, true); } item.track = track; item.receiver = receiver || item.receiver; item.reasons.add(reason); selectRemoteVideo(reason); } } function rememberRemoteStream(stream, reason) { if (!stream || !(stream instanceof MediaStream)) return; for (const track of stream.getAudioTracks()) { if (aiOutputTrackIds.has(track.id)) continue; let item = remoteAudio.get(track.id); if (!item) { item = { track, receiver: null, reasons: new Set(), bytesDelta: 0, packetsDelta: 0 }; remoteAudio.set(track.id, item); log(`audio stream candidate ${trackLabel(track)} (${reason})`, true); } item.track = track; item.reasons.add(reason); } for (const track of stream.getVideoTracks()) { let item = remoteVideo.get(track.id); if (!item) { item = { track, receiver: null, reasons: new Set() }; remoteVideo.set(track.id, item); log(`video stream candidate ${trackLabel(track)} (${reason})`, true); } item.track = track; item.reasons.add(reason); } selectRemoteAudio(reason); selectRemoteVideo(reason); } function selectRemoteAudio(reason) { const current = selectedRemoteAudio && remoteAudio.get(selectedRemoteAudio.id); if (isLive(selectedRemoteAudio) && !selectedRemoteAudio.muted && hasElementReason(current, STATIC_AUDIO_ELEMENT_REASON)) { return selectedRemoteAudio; } const item = Array.from(remoteAudio.values()).find((candidate) => ( isLive(candidate.track) && !candidate.track.muted && hasElementReason(candidate, STATIC_AUDIO_ELEMENT_REASON) )); if (!item) return null; if (selectedRemoteAudio !== item.track) { selectedRemoteAudio = item.track; log(`selected input audio (${reason}) from ${STATIC_AUDIO_ELEMENT_REASON}: ${trackLabel(selectedRemoteAudio)}`, true); attachAudioToAi(); } return selectedRemoteAudio; } function selectRemoteVideo(reason) { const current = selectedRemoteVideo && remoteVideo.get(selectedRemoteVideo.id); if (isLive(selectedRemoteVideo) && !selectedRemoteVideo.muted && hasElementReason(current, STATIC_VIDEO_ELEMENT_REASON)) { return selectedRemoteVideo; } const item = Array.from(remoteVideo.values()).find((candidate) => ( isLive(candidate.track) && !candidate.track.muted && hasElementReason(candidate, STATIC_VIDEO_ELEMENT_REASON) )); if (!item) return null; if (selectedRemoteVideo !== item.track) { selectedRemoteVideo = item.track; log(`selected input video (${reason}) from ${STATIC_VIDEO_ELEMENT_REASON}: ${trackLabel(selectedRemoteVideo)}`, true); attachVideoToAi(); } return selectedRemoteVideo; } function rememberBusinessSender(sender, track, reason) { if (!sender) return; const senderTrack = track || sender.track; if (senderTrack && senderTrack.kind !== 'audio') return; const id = sender.__aiNativeElementSenderId || `sender-${businessSenders.size + 1}`; sender.__aiNativeElementSenderId = id; let item = businessSenders.get(id); if (!item) { item = { id, sender, track: senderTrack || null, originalTrack: senderTrack || null, sources: new Set(), bytesDelta: 0, packetsDelta: 0 }; businessSenders.set(id, item); log(`business sender candidate ${id} (${reason}): ${trackLabel(senderTrack)}`, true); } item.sender = sender; item.track = sender.track || senderTrack || item.track; if (!isAiTakeover && senderTrack && senderTrack !== aiAudioTrack) item.originalTrack = senderTrack; item.sources.add(reason); wrapBusinessSender(sender); selectBusinessSender(reason); } function wrapBusinessSender(sender) { if (!sender || sender.__aiNativeElementWrapped) return; const originalReplaceTrack = sender.replaceTrack?.bind(sender); if (!originalReplaceTrack) return; sender.replaceTrack = async function (track) { const result = await originalReplaceTrack(track); rememberBusinessSender(sender, track, 'replaceTrack'); return result; }; sender.__aiNativeElementWrapped = true; } function isRemoteInputTrack(track) { return Boolean(track && remoteAudio.has(track.id) && !aiOutputTrackIds.has(track.id)); } function selectBusinessSender(reason) { if (selectedBusinessSender && !isRemoteInputTrack(selectedBusinessSender.track)) return selectedBusinessSender; const item = Array.from(businessSenders.values()).find((candidate) => { const track = candidate.sender.track || candidate.track || candidate.originalTrack; return candidate.sources.has('pc-addTrack') && !isRemoteInputTrack(track); }); if (!item) return null; selectedBusinessSender = item.sender; originalBusinessTrack = item.originalTrack || item.sender.track || originalBusinessTrack; log(`selected output sender (${reason}): ${item.id}, track=${trackLabel(item.sender.track)}`, true); attachAiOutputToBusiness(); return selectedBusinessSender; } function setVu(id, level) { const el = document.getElementById(id); if (el) el.style.width = `${Math.min(100, level * 2500)}%`; } function renderVu() { inputVu *= 0.88; outputVu *= 0.88; setVu('element-copilot-input-vu', inputVu); setVu('element-copilot-output-vu', outputVu); requestAnimationFrame(renderVu); } async function refreshStats() { await refreshInputStats(); await refreshOutputStats(); } async function refreshInputStats() { for (const item of remoteAudio.values()) { if (!item.receiver?.getStats) continue; const report = await item.receiver.getStats(); report.forEach((stat) => { const kind = stat.kind || stat.mediaType; if (stat.type !== 'inbound-rtp' || kind !== 'audio') return; const prev = inboundMemory.get(stat.id); const bytes = stat.bytesReceived || 0; const packets = stat.packetsReceived || 0; item.bytesReceived = bytes; item.packetsReceived = packets; item.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0; item.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0; inboundMemory.set(stat.id, { bytes, packets }); if (item.track === selectedRemoteAudio) { inputVu = Math.max(inputVu, Math.min(0.12, item.bytesDelta / 12000), Math.min(0.04, item.packetsDelta / 500)); } }); } } async function refreshOutputStats() { for (const item of businessSenders.values()) { if (!item.sender?.getStats) continue; item.track = item.sender.track || item.track; const report = await item.sender.getStats(); report.forEach((stat) => { const kind = stat.kind || stat.mediaType; if (stat.type !== 'outbound-rtp' || kind !== 'audio') return; const key = `${item.id}:${stat.id}`; const prev = outboundMemory.get(key); const bytes = stat.bytesSent || 0; const packets = stat.packetsSent || 0; item.bytesSent = bytes; item.packetsSent = packets; item.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0; item.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0; outboundMemory.set(key, { bytes, packets }); if (item.sender === selectedBusinessSender) { outputVu = Math.max(outputVu, Math.min(0.12, item.bytesDelta / 12000), Math.min(0.04, item.packetsDelta / 500)); } }); } } async function outboundBytes(sender) { if (!sender?.getStats) return null; let total = 0; const report = await sender.getStats(); report.forEach((stat) => { const kind = stat.kind || stat.mediaType; if (stat.type === 'outbound-rtp' && kind === 'audio') total += stat.bytesSent || 0; }); return total; } function aiState() { if (!aiPc) return 'pc=null'; return `conn=${aiPc.connectionState},ice=${aiPc.iceConnectionState},signaling=${aiPc.signalingState},remote=${aiPc.remoteDescription?.type || 'none'}`; } function aiReady() { return Boolean(aiPc?.remoteDescription && ( aiPc.connectionState === 'connected' || aiPc.iceConnectionState === 'connected' || aiPc.iceConnectionState === 'completed' )); } function waitForAiReady(timeoutMs = 10000) { if (aiReady()) return Promise.resolve(true); return new Promise((resolve) => { const startedAt = Date.now(); const timer = setInterval(() => { if (aiReady()) { clearInterval(timer); resolve(true); } else if (Date.now() - startedAt >= timeoutMs) { clearInterval(timer); resolve(false); } }, 250); }); } function checkAiInput(track) { if (inputHealthTimer) clearTimeout(inputHealthTimer); inputHealthTimer = setTimeout(async () => { if (!aiAudioSender || aiAudioSender.track !== track) return; await waitForAiReady(); const before = await outboundBytes(aiAudioSender); await new Promise((resolve) => setTimeout(resolve, 2500)); const after = await outboundBytes(aiAudioSender); log(`AI input packet check: before=${before}, after=${after}, ${aiState()}`, true); }, 500); } function checkAiOutput(track) { if (outputHealthTimer) clearTimeout(outputHealthTimer); outputHealthTimer = setTimeout(async () => { if (!selectedBusinessSender || selectedBusinessSender.track !== track) return; const before = await outboundBytes(selectedBusinessSender); await new Promise((resolve) => setTimeout(resolve, 2500)); const after = await outboundBytes(selectedBusinessSender); log(`AI output packet check: before=${before}, after=${after}, sender=${trackLabel(selectedBusinessSender.track)}`, true); }, 800); } async function attachAudioToAi() { if (!shouldSendAudioToAi()) return false; if (!aiAudioSender || !isLive(selectedRemoteAudio)) return false; if (aiAudioSender.track === selectedRemoteAudio) return true; await aiAudioSender.replaceTrack(selectedRemoteAudio); log(`AI input uses ${trackLabel(selectedRemoteAudio)} from ${STATIC_AUDIO_ELEMENT_REASON}`, true); checkAiInput(selectedRemoteAudio); return true; } async function attachVideoToAi() { if (!shouldSendVideoToAi()) return false; if (!aiVideoSender || !isLive(selectedRemoteVideo)) return false; if (aiVideoSender.track === selectedRemoteVideo) return true; await aiVideoSender.replaceTrack(selectedRemoteVideo); log(`AI video uses ${trackLabel(selectedRemoteVideo)} from ${STATIC_VIDEO_ELEMENT_REASON}`, true); return true; } async function attachAiOutputToBusiness() { selectBusinessSender('before-output'); if (!isAiTakeover || !selectedBusinessSender || !isLive(aiAudioTrack)) { log(`AI output waiting: takeover=${isAiTakeover}, sender=${Boolean(selectedBusinessSender)}, aiTrack=${trackLabel(aiAudioTrack)}`); return false; } if (selectedBusinessSender.track === aiAudioTrack) return true; await selectedBusinessSender.replaceTrack(aiAudioTrack); aiAttached = true; log(`business sender uses AI audio ${trackLabel(aiAudioTrack)}`, true); checkAiOutput(aiAudioTrack); return true; } async function restoreBusinessAudio() { if (!selectedBusinessSender || !aiAttached) return; await selectedBusinessSender.replaceTrack(originalBusinessTrack || null); aiAttached = false; log(`business sender restored ${trackLabel(originalBusinessTrack)}`, true); } function wrapTrackListener(listener) { if (!listener || listener.__aiNativeElementWrapped) return listener; const wrapped = function (event) { const ownerPc = this instanceof RTCPeerConnection ? this : event?.currentTarget; if (!isInternalPc(ownerPc) && event?.track) { rememberRemoteReceiver(event.receiver, 'pc-track'); for (const stream of event.streams || []) rememberRemoteStream(stream, 'pc-track-stream'); } if (typeof listener === 'function') return listener.apply(this, arguments); if (listener?.handleEvent) return listener.handleEvent(event); return undefined; }; wrapped.__aiNativeElementWrapped = true; return wrapped; } RTCPeerConnection.prototype.addEventListener = function (type, listener, options) { if (type === 'track') return originalAddEventListener.call(this, type, wrapTrackListener(listener), options); return originalAddEventListener.call(this, type, listener, options); }; try { const descriptor = Object.getOwnPropertyDescriptor(RTCPeerConnection.prototype, 'ontrack'); Object.defineProperty(RTCPeerConnection.prototype, 'ontrack', { configurable: true, enumerable: true, get() { return descriptor?.get ? descriptor.get.call(this) : this.__aiNativeElementOnTrack || null; }, set(listener) { const wrapped = wrapTrackListener(listener); this.__aiNativeElementOnTrack = wrapped; if (descriptor?.set) descriptor.set.call(this, wrapped); else this.addEventListener('track', wrapped); }, }); } catch (error) { console.warn('[AI Native Element] failed to wrap ontrack', error); } RTCPeerConnection.prototype.addTrack = function (track, ...streams) { const sender = originalAddTrack.call(this, track, ...streams); if (!isInternalPc(this) && track?.kind === 'audio') rememberBusinessSender(sender, track, 'pc-addTrack'); return sender; }; if (originalAddTransceiver) { RTCPeerConnection.prototype.addTransceiver = function (trackOrKind, init) { const transceiver = originalAddTransceiver.call(this, trackOrKind, init); if (!isInternalPc(this)) { const kind = trackOrKind instanceof MediaStreamTrack ? trackOrKind.kind : trackOrKind; if (kind === 'audio') { const track = trackOrKind instanceof MediaStreamTrack ? trackOrKind : transceiver.sender?.track; rememberBusinessSender(transceiver.sender, track, 'pc-addTransceiver'); } } return transceiver; }; } RTCPeerConnection.prototype.setRemoteDescription = function () { const result = originalSetRemoteDescription.apply(this, arguments); if (!isInternalPc(this)) { Promise.resolve(result).then(() => { for (const receiver of this.getReceivers?.() || []) rememberRemoteReceiver(receiver, 'receiver-after-srd'); for (const sender of this.getSenders?.() || []) rememberBusinessSender(sender, sender.track, 'sender-after-srd'); }).catch(() => {}); } return result; }; if (mediaSrcObjectDescriptor?.set) { Object.defineProperty(HTMLMediaElement.prototype, 'srcObject', { configurable: true, enumerable: mediaSrcObjectDescriptor.enumerable, get() { return mediaSrcObjectDescriptor.get.call(this); }, set(stream) { if (!this.__aiNativeElementCopilotElement) { const reason = elementReasonFor(this) || `${this.tagName.toLowerCase()}-srcObject`; rememberRemoteStream(stream, reason); } return mediaSrcObjectDescriptor.set.call(this, stream); }, }); } function scanMediaElements() { document.querySelectorAll('audio').forEach((element, index) => { if (element.__aiNativeElementCopilotElement) return; rememberRemoteStream(element.srcObject, `element-${index}:audio`); }); document.querySelectorAll('video').forEach((element, index) => { if (element.__aiNativeElementCopilotElement) return; rememberRemoteStream(element.srcObject, `element-${index}:video`); }); selectRemoteAudio('media-scan'); selectRemoteVideo('media-scan'); } setInterval(scanMediaElements, 1000); async function connectAi(signalingUrl, assistantId) { disconnectAi(); const id = pcId(); aiPc = new RTCPeerConnection({ iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] }); markInternalPc(aiPc); aiPc.onconnectionstatechange = () => log(`AI PC connectionState=${aiPc.connectionState}`); aiPc.oniceconnectionstatechange = () => log(`AI PC iceConnectionState=${aiPc.iceConnectionState}`); aiDc = aiPc.createDataChannel('chat'); aiDc.onopen = () => { setStatus('connected'); log('AI data channel open', true); aiDc.send(JSON.stringify({ type: 'client-ready' })); }; aiDc.onclose = () => setStatus('disconnected'); aiDc.onmessage = (event) => { try { const msg = JSON.parse(event.data); const content = msg.delta || msg.content; if (msg.type === 'transcript') { logUserTranscript(content); } else if (msg.type === 'assistant-text-start') { startAssistantLine(); } else if (msg.type === 'assistant-text-delta') { appendAssistantDelta(content); } else if (msg.type === 'assistant-text-end') { finishAssistantLine(); } else { log(`data channel message: ${msg.type}`); } } catch { appendChatLine('DATA', event.data); } }; scanMediaElements(); selectRemoteAudio('connect'); selectRemoteVideo('connect'); selectBusinessSender('connect'); const sendAudio = shouldSendAudioToAi(); const audioTransceiver = sendAudio && isLive(selectedRemoteAudio) ? aiPc.addTransceiver(selectedRemoteAudio, { direction: 'sendrecv' }) : aiPc.addTransceiver('audio', { direction: sendAudio ? 'sendrecv' : 'recvonly' }); aiAudioSender = audioTransceiver.sender; log(!sendAudio ? 'AI audio uplink disabled by panel setting' : isLive(selectedRemoteAudio) ? `AI input starts with ${trackLabel(selectedRemoteAudio)} (${STATIC_AUDIO_ELEMENT_REASON})` : `AI input waiting for ${STATIC_AUDIO_ELEMENT_REASON}`, true); const sendVideo = shouldSendVideoToAi(); const shouldSendVideo = sendVideo && isLive(selectedRemoteVideo) && !selectedRemoteVideo.muted; if (shouldSendVideo) { const videoTransceiver = aiPc.addTransceiver(selectedRemoteVideo, { direction: 'sendonly' }); aiVideoSender = videoTransceiver.sender; log(`AI video starts with ${trackLabel(selectedRemoteVideo)} (${STATIC_VIDEO_ELEMENT_REASON})`); } else { aiVideoSender = null; log(sendVideo ? `AI video disabled: waiting for live track at ${STATIC_VIDEO_ELEMENT_REASON}` : 'AI video uplink disabled by panel setting'); } aiPc.ontrack = (event) => { if (event.track.kind !== 'audio') return; aiOutputTrackIds.add(event.track.id); remoteAudio.delete(event.track.id); aiAudioTrack = event.track; log(`received AI audio ${trackLabel(aiAudioTrack)}`, true); aiAudioTrack.onunmute = () => { log(`AI audio unmuted ${trackLabel(aiAudioTrack)}`); attachAiOutputToBusiness(); }; attachAiOutputToBusiness(); }; aiPc.onicecandidate = (event) => { if (aiWs?.readyState !== WebSocket.OPEN) return; aiWs.send(JSON.stringify({ type: 'ice-candidate', payload: { pc_id: id, candidate: event.candidate ? { candidate: event.candidate.candidate, sdpMid: event.candidate.sdpMid, sdpMLineIndex: event.candidate.sdpMLineIndex, } : null, }, })); }; setStatus('connecting'); aiWs = new WebSocket(signalingUrl); aiWs.onopen = async () => { const offer = await aiPc.createOffer(); await aiPc.setLocalDescription(offer); aiWs.send(JSON.stringify({ type: 'offer', payload: { pc_id: id, sdp: aiPc.localDescription.sdp, type: aiPc.localDescription.type, assistant_id: assistantId, vision_enabled: shouldSendVideo, }, })); log('sent AI WebRTC offer', true); }; aiWs.onmessage = async (event) => { try { const msg = JSON.parse(event.data); if (msg.type === 'answer') { await aiPc.setRemoteDescription({ type: 'answer', sdp: msg.payload.sdp }); log(`AI WebRTC answer applied; ${aiState()}`, true); } else if (msg.type === 'ice-candidate' && msg.payload?.candidate) { await aiPc.addIceCandidate(msg.payload.candidate).catch(() => {}); } else if (msg.type === 'error') { throw new Error(msg.payload?.message || 'server error'); } } catch (error) { log(`signaling error: ${error.message || error}`); } }; aiWs.onerror = () => { setStatus('failed'); log('AI signaling websocket error'); }; aiWs.onclose = () => setStatus('disconnected'); attachAudioToAi(); attachVideoToAi(); } function disconnectAi() { try { aiDc?.close(); } catch {} try { aiPc?.close(); } catch {} try { aiWs?.close(); } catch {} aiDc = null; aiPc = null; aiWs = null; aiAudioSender = null; aiVideoSender = null; aiAudioTrack = null; if (inputRecorder?.state === 'recording') stopInputRecording(); setStatus('disconnected'); } function setStatus(status) { const el = document.getElementById('element-copilot-status'); if (!el) return; el.textContent = { disconnected: '未连接', connecting: '连接中', connected: '已连接', failed: '连接失败', }[status] || status; } function updateModeButton() { const button = document.getElementById('element-copilot-toggle'); if (!button) return; if (isAiTakeover) { button.textContent = '当前:AI 接管|点击切回人工'; button.style.background = '#8b5cf6'; } else { button.textContent = '当前:人工接管|点击切换 AI'; button.style.background = '#10b981'; } } window.__aiNativeElementDump = function () { return { staticAudioElement: STATIC_AUDIO_ELEMENT_REASON, staticVideoElement: STATIC_VIDEO_ELEMENT_REASON, selectedInput: trackLabel(selectedRemoteAudio), selectedVideo: trackLabel(selectedRemoteVideo), aiVideoSenderTrack: trackLabel(aiVideoSender?.track), selectedOutput: trackLabel(selectedBusinessSender?.track), originalOutput: trackLabel(originalBusinessTrack), aiAudio: trackLabel(aiAudioTrack), aiAttached, aiState: aiState(), recording: inputRecorder?.state === 'recording' ? { track: trackLabel(inputRecordingTrack), seconds: ((Date.now() - inputRecordingStartedAt) / 1000).toFixed(1), chunks: inputRecordingChunks.length, } : null, inputs: Array.from(remoteAudio.values()).map((item) => ({ selected: item.track === selectedRemoteAudio, id: item.track.id, muted: item.track.muted, live: item.track.readyState, hasReceiver: Boolean(item.receiver), matchesStaticElement: hasElementReason(item, STATIC_AUDIO_ELEMENT_REASON), bytesReceived: item.bytesReceived || 0, packetsReceived: item.packetsReceived || 0, bytesDelta: item.bytesDelta, packetsDelta: item.packetsDelta, reasons: Array.from(item.reasons), })), videos: Array.from(remoteVideo.values()).map((item) => ({ selected: item.track === selectedRemoteVideo, id: item.track.id, muted: item.track.muted, live: item.track.readyState, hasReceiver: Boolean(item.receiver), matchesStaticElement: hasElementReason(item, STATIC_VIDEO_ELEMENT_REASON), reasons: Array.from(item.reasons), })), outputs: Array.from(businessSenders.values()).map((item) => ({ selected: item.sender === selectedBusinessSender, id: item.id, track: trackLabel(item.sender.track || item.track), originalTrack: trackLabel(item.originalTrack), pcAddTrack: item.sources.has('pc-addTrack'), isRemoteInput: isRemoteInputTrack(item.sender.track || item.track), bytesSent: item.bytesSent, packetsSent: item.packetsSent, bytesDelta: item.bytesDelta, packetsDelta: item.packetsDelta, sources: Array.from(item.sources), })), }; }; function renderPanel() { if (!document.body || document.getElementById('element-copilot-panel')) return; const panel = document.createElement('div'); panel.id = 'element-copilot-panel'; panel.style.cssText = [ 'position:fixed', 'top:20px', 'left:20px', 'width:360px', 'z-index:999999', 'box-sizing:border-box', 'padding:16px', 'border-radius:10px', 'background:rgba(17,24,39,.96)', 'color:white', 'font-family:system-ui,-apple-system,BlinkMacSystemFont,sans-serif', 'box-shadow:0 12px 32px rgba(0,0,0,.35)', ].join(';'); panel.innerHTML = `