diff --git a/scripts/copilot-native-static.js b/scripts/copilot-native-static.js new file mode 100644 index 0000000..bbee563 --- /dev/null +++ b/scripts/copilot-native-static.js @@ -0,0 +1,876 @@ +// ==UserScript== +// @name 12345 AI Native Static Copilot +// @namespace http://tampermonkey.net/ +// @version 1.0 +// @description Native-track bridge with static input/output selection rules. +// @author You +// @match *://yhy-yw.22sj.cn:*/* +// @grant none +// @run-at document-start +// ==/UserScript== + +(function () { + 'use strict'; + + const DEFAULT_SIGNALING_URL = 'ws://182.92.86.220:8000/ws/voice'; + + const originalAddEventListener = RTCPeerConnection.prototype.addEventListener; + const originalAddTrack = RTCPeerConnection.prototype.addTrack; + const originalAddTransceiver = RTCPeerConnection.prototype.addTransceiver; + const originalSetRemoteDescription = RTCPeerConnection.prototype.setRemoteDescription; + const mediaSrcObjectDescriptor = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, 'srcObject'); + + const remoteAudio = new Map(); + const remoteVideo = new Map(); + const businessSenders = new Map(); + const aiOutputTrackIds = new Set(); + const inboundMemory = new Map(); + const outboundMemory = new Map(); + const internalPcs = new WeakSet(); + const ENABLE_VIDEO_UPLINK = true; + + let selectedRemoteAudio = null; + let selectedRemoteVideo = null; + let selectedBusinessSender = null; + let originalBusinessTrack = null; + let aiAudioTrack = null; + + let aiPc = null; + let aiWs = null; + let aiDc = null; + let aiAudioSender = null; + let aiVideoSender = null; + let isAiTakeover = false; + let aiAttached = false; + let inputVu = 0; + let outputVu = 0; + let inputHealthTimer = null; + let outputHealthTimer = null; + let assistantLine = null; + let assistantBuffer = ''; + let inputRecorder = null; + let inputRecordingChunks = []; + let inputRecordingTrack = null; + let inputRecordingStartedAt = 0; + + function log(message) { + console.log(`[AI Native Static] ${message}`); + } + + function appendChatLine(role, text, important = false) { + const box = document.getElementById('static-copilot-log'); + if (!box) return; + if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = ''; + const row = document.createElement('div'); + row.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] ${role}: ${text}`; + row.style.marginBottom = '4px'; + if (important) row.style.color = '#fcd34d'; + box.appendChild(row); + while (box.children.length > 120) box.removeChild(box.firstChild); + box.scrollTop = box.scrollHeight; + return row; + } + + function startAssistantLine() { + assistantBuffer = ''; + assistantLine = null; + } + + function appendAssistantDelta(delta) { + if (!delta) return; + const box = document.getElementById('static-copilot-log'); + if (!box) { + console.log(`[AI Native Static] AI: ${delta}`); + return; + } + if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = ''; + if (!assistantLine) assistantLine = appendChatLine('AI', ''); + assistantBuffer += delta; + assistantLine.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] AI: ${assistantBuffer}`; + box.scrollTop = box.scrollHeight; + } + + function finishAssistantLine() { + assistantLine = null; + assistantBuffer = ''; + } + + function logUserTranscript(text) { + if (!text?.trim()) return; + finishAssistantLine(); + appendChatLine('用户', text.trim(), true); + } + + function recorderMimeType() { + if (window.MediaRecorder?.isTypeSupported?.('audio/webm;codecs=opus')) return 'audio/webm;codecs=opus'; + if (window.MediaRecorder?.isTypeSupported?.('audio/webm')) return 'audio/webm'; + return ''; + } + + function recordingFileName(track) { + const stamp = new Date().toISOString().replace(/[:.]/g, '-'); + const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track'; + return `ai-input-${trackPart}-${stamp}.webm`; + } + + async function startInputRecording() { + if (inputRecorder?.state === 'recording') return; + selectRemoteAudio('record'); + if (!isLive(selectedRemoteAudio)) { + appendChatLine('录音', '没有可录制的远端输入音轨', true); + return; + } + if (!window.MediaRecorder) { + appendChatLine('录音', '当前浏览器不支持 MediaRecorder', true); + return; + } + + inputRecordingChunks = []; + inputRecordingTrack = selectedRemoteAudio; + inputRecordingStartedAt = Date.now(); + const stream = new MediaStream([selectedRemoteAudio]); + const mimeType = recorderMimeType(); + inputRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); + inputRecorder.ondataavailable = (event) => { + if (event.data?.size > 0) inputRecordingChunks.push(event.data); + }; + inputRecorder.onstop = () => { + const blob = new Blob(inputRecordingChunks, { type: inputRecorder?.mimeType || 'audio/webm' }); + const url = URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = url; + link.download = recordingFileName(inputRecordingTrack); + document.body.appendChild(link); + link.click(); + link.remove(); + setTimeout(() => URL.revokeObjectURL(url), 30000); + + const seconds = ((Date.now() - inputRecordingStartedAt) / 1000).toFixed(1); + appendChatLine('录音', `已保存 ${link.download}（${seconds}s，${(blob.size / 1024).toFixed(1)} KB）`, true); + inputRecorder = null; + inputRecordingChunks = []; + inputRecordingTrack = null; + updateRecordButton(); + }; + inputRecorder.start(250); + appendChatLine('录音', `开始录制 AI 输入：${trackLabel(inputRecordingTrack)}`, true); + updateRecordButton(); + } + + function stopInputRecording() { + if (inputRecorder?.state === 'recording') inputRecorder.stop(); + } + + function toggleInputRecording() { + if (inputRecorder?.state === 'recording') stopInputRecording(); + else startInputRecording(); + } + + function updateRecordButton() { + const button = document.getElementById('static-copilot-record'); + if (!button) return; + const recording = inputRecorder?.state === 'recording'; + button.textContent = recording ? '停止录制 AI 输入' : '录制 AI 输入'; + button.style.background = recording ? '#dc2626' : '#2563eb'; + } + + function pcId() { + const bytes = new Uint8Array(16); + crypto.getRandomValues(bytes); + return 'PC-' + Array.from(bytes).map((b) => b.toString(16).padStart(2, '0')).join(''); + } + + function isInternalPc(pc) { + return Boolean(pc && (pc.__aiNativeStaticInternal || internalPcs.has(pc) || pc === aiPc)); + } + + function isLive(track) { + return track && track.readyState === 'live'; + } + + function trackLabel(track) { + if (!track) return 'none'; + return `${track.id}${track.muted ? ' muted' : ''}`; + } + + function markInternalPc(pc) { + if (!pc) return; + pc.__aiNativeStaticInternal = true; + internalPcs.add(pc); + } + + function rememberRemoteReceiver(receiver, reason) { + const track = receiver?.track; + if (!isLive(track) || isInternalPc(receiver?.transport)) return; + + if (track.kind === 'audio' && !aiOutputTrackIds.has(track.id)) { + let item = remoteAudio.get(track.id); + if (!item) { + item = { track, receiver, reasons: new Set(), bytesDelta: 0, packetsDelta: 0 }; + remoteAudio.set(track.id, item); + log(`audio candidate ${trackLabel(track)} (${reason})`, true); + } + item.track = track; + item.receiver = receiver || item.receiver; + item.reasons.add(reason); + selectRemoteAudio(reason); + } + + if (track.kind === 'video') { + let item = remoteVideo.get(track.id); + if (!item) { + item = { track, receiver, reasons: new Set() }; + remoteVideo.set(track.id, item); + log(`video candidate ${trackLabel(track)} (${reason})`, true); + } + item.track = track; + item.receiver = receiver || item.receiver; + item.reasons.add(reason); + if (!selectedRemoteVideo || selectedRemoteVideo.readyState !== 'live') { + selectedRemoteVideo = track; + attachVideoToAi(); + } + } + } + + function rememberRemoteStream(stream, reason) { + if (!stream || !(stream instanceof MediaStream)) return; + for (const track of stream.getAudioTracks()) { + if (aiOutputTrackIds.has(track.id)) continue; + let item = remoteAudio.get(track.id); + if (!item) { + item = { track, receiver: null, reasons: new Set(), bytesDelta: 0, packetsDelta: 0 }; + remoteAudio.set(track.id, item); + log(`audio stream candidate ${trackLabel(track)} (${reason})`, true); + } + item.track = track; + item.reasons.add(reason); + } + for (const track of stream.getVideoTracks()) { + let item = remoteVideo.get(track.id); + if (!item) { + item = { track, receiver: null, reasons: new Set() }; + remoteVideo.set(track.id, item); + log(`video stream candidate ${trackLabel(track)} (${reason})`, true); + } + item.track = track; + item.reasons.add(reason); + if (!selectedRemoteVideo || selectedRemoteVideo.readyState !== 'live') selectedRemoteVideo = track; + } + selectRemoteAudio(reason); + attachVideoToAi(); + } + + function selectRemoteAudio(reason) { + if (isLive(selectedRemoteAudio) && !selectedRemoteAudio.muted) return selectedRemoteAudio; + + const item = Array.from(remoteAudio.values()) + .find((candidate) => isLive(candidate.track) && !candidate.track.muted && candidate.receiver); + if (!item) return null; + + selectedRemoteAudio = item.track; + log(`selected input audio (${reason}): ${trackLabel(selectedRemoteAudio)}`, true); + attachAudioToAi(); + return selectedRemoteAudio; + } + + function rememberBusinessSender(sender, track, reason) { + if (!sender) return; + const senderTrack = track || sender.track; + if (senderTrack && senderTrack.kind !== 'audio') return; + + const id = sender.__aiNativeStaticSenderId || `sender-${businessSenders.size + 1}`; + sender.__aiNativeStaticSenderId = id; + + let item = businessSenders.get(id); + if (!item) { + item = { id, sender, track: senderTrack || null, originalTrack: senderTrack || null, sources: new Set(), bytesDelta: 0, packetsDelta: 0 }; + businessSenders.set(id, item); + log(`business sender candidate ${id} (${reason}): ${trackLabel(senderTrack)}`, true); + } + item.sender = sender; + item.track = sender.track || senderTrack || item.track; + if (!isAiTakeover && senderTrack && senderTrack !== aiAudioTrack) item.originalTrack = senderTrack; + item.sources.add(reason); + wrapBusinessSender(sender); + selectBusinessSender(reason); + } + + function wrapBusinessSender(sender) { + if (!sender || sender.__aiNativeStaticWrapped) return; + const originalReplaceTrack = sender.replaceTrack?.bind(sender); + if (!originalReplaceTrack) return; + + sender.replaceTrack = async function (track) { + const result = await originalReplaceTrack(track); + rememberBusinessSender(sender, track, 'replaceTrack'); + return result; + }; + sender.__aiNativeStaticWrapped = true; + } + + function isRemoteInputTrack(track) { + return Boolean(track && remoteAudio.has(track.id) && !aiOutputTrackIds.has(track.id)); + } + + function selectBusinessSender(reason) { + if (selectedBusinessSender && !isRemoteInputTrack(selectedBusinessSender.track)) return selectedBusinessSender; + + const item = Array.from(businessSenders.values()).find((candidate) => { + const track = candidate.sender.track || candidate.track || candidate.originalTrack; + return candidate.sources.has('pc-addTrack') && !isRemoteInputTrack(track); + }); + if (!item) return null; + + selectedBusinessSender = item.sender; + originalBusinessTrack = item.originalTrack || item.sender.track || originalBusinessTrack; + log(`selected output sender (${reason}): ${item.id}, track=${trackLabel(item.sender.track)}`, true); + attachAiOutputToBusiness(); + return selectedBusinessSender; + } + + function setVu(id, level) { + const el = document.getElementById(id); + if (el) el.style.width = `${Math.min(100, level * 2500)}%`; + } + + function renderVu() { + inputVu *= 0.88; + outputVu *= 0.88; + setVu('static-copilot-input-vu', inputVu); + setVu('static-copilot-output-vu', outputVu); + requestAnimationFrame(renderVu); + } + + async function refreshStats() { + await refreshInputStats(); + await refreshOutputStats(); + } + + async function refreshInputStats() { + for (const item of remoteAudio.values()) { + if (!item.receiver?.getStats) continue; + const report = await item.receiver.getStats(); + report.forEach((stat) => { + const kind = stat.kind || stat.mediaType; + if (stat.type !== 'inbound-rtp' || kind !== 'audio') return; + const prev = inboundMemory.get(stat.id); + const bytes = stat.bytesReceived || 0; + const packets = stat.packetsReceived || 0; + item.bytesReceived = bytes; + item.packetsReceived = packets; + item.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0; + item.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0; + inboundMemory.set(stat.id, { bytes, packets }); + if (item.track === selectedRemoteAudio) { + inputVu = Math.max(inputVu, Math.min(0.12, item.bytesDelta / 12000), Math.min(0.04, item.packetsDelta / 500)); + } + }); + } + } + + async function refreshOutputStats() { + for (const item of businessSenders.values()) { + if (!item.sender?.getStats) continue; + item.track = item.sender.track || item.track; + const report = await item.sender.getStats(); + report.forEach((stat) => { + const kind = stat.kind || stat.mediaType; + if (stat.type !== 'outbound-rtp' || kind !== 'audio') return; + const key = `${item.id}:${stat.id}`; + const prev = outboundMemory.get(key); + const bytes = stat.bytesSent || 0; + const packets = stat.packetsSent || 0; + item.bytesSent = bytes; + item.packetsSent = packets; + item.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0; + item.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0; + outboundMemory.set(key, { bytes, packets }); + if (item.sender === selectedBusinessSender) { + outputVu = Math.max(outputVu, Math.min(0.12, item.bytesDelta / 12000), Math.min(0.04, item.packetsDelta / 500)); + } + }); + } + } + + async function outboundBytes(sender) { + if (!sender?.getStats) return null; + let total = 0; + const report = await sender.getStats(); + report.forEach((stat) => { + const kind = stat.kind || stat.mediaType; + if (stat.type === 'outbound-rtp' && kind === 'audio') total += stat.bytesSent || 0; + }); + return total; + } + + function aiState() { + if (!aiPc) return 'pc=null'; + return `conn=${aiPc.connectionState},ice=${aiPc.iceConnectionState},signaling=${aiPc.signalingState},remote=${aiPc.remoteDescription?.type || 'none'}`; + } + + function aiReady() { + return Boolean(aiPc?.remoteDescription && ( + aiPc.connectionState === 'connected' + || aiPc.iceConnectionState === 'connected' + || aiPc.iceConnectionState === 'completed' + )); + } + + function waitForAiReady(timeoutMs = 10000) { + if (aiReady()) return Promise.resolve(true); + return new Promise((resolve) => { + const startedAt = Date.now(); + const timer = setInterval(() => { + if (aiReady()) { + clearInterval(timer); + resolve(true); + } else if (Date.now() - startedAt >= timeoutMs) { + clearInterval(timer); + resolve(false); + } + }, 250); + }); + } + + function checkAiInput(track) { + if (inputHealthTimer) clearTimeout(inputHealthTimer); + inputHealthTimer = setTimeout(async () => { + if (!aiAudioSender || aiAudioSender.track !== track) return; + await waitForAiReady(); + const before = await outboundBytes(aiAudioSender); + await new Promise((resolve) => setTimeout(resolve, 2500)); + const after = await outboundBytes(aiAudioSender); + log(`AI input packet check: before=${before}, after=${after}, ${aiState()}`, true); + }, 500); + } + + function checkAiOutput(track) { + if (outputHealthTimer) clearTimeout(outputHealthTimer); + outputHealthTimer = setTimeout(async () => { + if (!selectedBusinessSender || selectedBusinessSender.track !== track) return; + const before = await outboundBytes(selectedBusinessSender); + await new Promise((resolve) => setTimeout(resolve, 2500)); + const after = await outboundBytes(selectedBusinessSender); + log(`AI output packet check: before=${before}, after=${after}, sender=${trackLabel(selectedBusinessSender.track)}`, true); + }, 800); + } + + async function attachAudioToAi() { + if (!aiAudioSender || !isLive(selectedRemoteAudio)) return false; + if (aiAudioSender.track === selectedRemoteAudio) return true; + await aiAudioSender.replaceTrack(selectedRemoteAudio); + log(`AI input uses ${trackLabel(selectedRemoteAudio)}`, true); + checkAiInput(selectedRemoteAudio); + return true; + } + + async function attachVideoToAi() { + if (!aiVideoSender || !isLive(selectedRemoteVideo)) return false; + if (aiVideoSender.track === selectedRemoteVideo) return true; + await aiVideoSender.replaceTrack(selectedRemoteVideo); + log(`AI video uses ${trackLabel(selectedRemoteVideo)}`, true); + return true; + } + + async function attachAiOutputToBusiness() { + selectBusinessSender('before-output'); + if (!isAiTakeover || !selectedBusinessSender || !isLive(aiAudioTrack)) { + log(`AI output waiting: takeover=${isAiTakeover}, sender=${Boolean(selectedBusinessSender)}, aiTrack=${trackLabel(aiAudioTrack)}`); + return false; + } + if (selectedBusinessSender.track === aiAudioTrack) return true; + await selectedBusinessSender.replaceTrack(aiAudioTrack); + aiAttached = true; + log(`business sender uses AI audio ${trackLabel(aiAudioTrack)}`, true); + checkAiOutput(aiAudioTrack); + return true; + } + + async function restoreBusinessAudio() { + if (!selectedBusinessSender || !aiAttached) return; + await selectedBusinessSender.replaceTrack(originalBusinessTrack || null); + aiAttached = false; + log(`business sender restored ${trackLabel(originalBusinessTrack)}`, true); + } + + function wrapTrackListener(listener) { + if (!listener || listener.__aiNativeStaticWrapped) return listener; + const wrapped = function (event) { + const ownerPc = this instanceof RTCPeerConnection ? this : event?.currentTarget; + if (!isInternalPc(ownerPc) && event?.track) { + rememberRemoteReceiver(event.receiver, 'pc-track'); + for (const stream of event.streams || []) rememberRemoteStream(stream, 'pc-track-stream'); + } + if (typeof listener === 'function') return listener.apply(this, arguments); + if (listener?.handleEvent) return listener.handleEvent(event); + return undefined; + }; + wrapped.__aiNativeStaticWrapped = true; + return wrapped; + } + + RTCPeerConnection.prototype.addEventListener = function (type, listener, options) { + if (type === 'track') return originalAddEventListener.call(this, type, wrapTrackListener(listener), options); + return originalAddEventListener.call(this, type, listener, options); + }; + + try { + const descriptor = Object.getOwnPropertyDescriptor(RTCPeerConnection.prototype, 'ontrack'); + Object.defineProperty(RTCPeerConnection.prototype, 'ontrack', { + configurable: true, + enumerable: true, + get() { + return descriptor?.get ? descriptor.get.call(this) : this.__aiNativeStaticOnTrack || null; + }, + set(listener) { + const wrapped = wrapTrackListener(listener); + this.__aiNativeStaticOnTrack = wrapped; + if (descriptor?.set) descriptor.set.call(this, wrapped); + else this.addEventListener('track', wrapped); + }, + }); + } catch (error) { + console.warn('[AI Native Static] failed to wrap ontrack', error); + } + + RTCPeerConnection.prototype.addTrack = function (track, ...streams) { + const sender = originalAddTrack.call(this, track, ...streams); + if (!isInternalPc(this) && track?.kind === 'audio') rememberBusinessSender(sender, track, 'pc-addTrack'); + return sender; + }; + + if (originalAddTransceiver) { + RTCPeerConnection.prototype.addTransceiver = function (trackOrKind, init) { + const transceiver = originalAddTransceiver.call(this, trackOrKind, init); + if (!isInternalPc(this)) { + const kind = trackOrKind instanceof MediaStreamTrack ? trackOrKind.kind : trackOrKind; + if (kind === 'audio') { + const track = trackOrKind instanceof MediaStreamTrack ? trackOrKind : transceiver.sender?.track; + rememberBusinessSender(transceiver.sender, track, 'pc-addTransceiver'); + } + } + return transceiver; + }; + } + + RTCPeerConnection.prototype.setRemoteDescription = function () { + const result = originalSetRemoteDescription.apply(this, arguments); + if (!isInternalPc(this)) { + Promise.resolve(result).then(() => { + for (const receiver of this.getReceivers?.() || []) rememberRemoteReceiver(receiver, 'receiver-after-srd'); + for (const sender of this.getSenders?.() || []) rememberBusinessSender(sender, sender.track, 'sender-after-srd'); + }).catch(() => {}); + } + return result; + }; + + if (mediaSrcObjectDescriptor?.set) { + Object.defineProperty(HTMLMediaElement.prototype, 'srcObject', { + configurable: true, + enumerable: mediaSrcObjectDescriptor.enumerable, + get() { + return mediaSrcObjectDescriptor.get.call(this); + }, + set(stream) { + if (!this.__aiNativeStaticElement) rememberRemoteStream(stream, `${this.tagName.toLowerCase()}-srcObject`); + return mediaSrcObjectDescriptor.set.call(this, stream); + }, + }); + } + + setInterval(() => { + for (const element of document.querySelectorAll('audio, video')) { + if (!element.__aiNativeStaticElement) rememberRemoteStream(element.srcObject, 'media-scan'); + } + }, 1000); + + async function connectAi(signalingUrl, assistantId) { + disconnectAi(); + + const id = pcId(); + aiPc = new RTCPeerConnection({ iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] }); + markInternalPc(aiPc); + aiPc.onconnectionstatechange = () => log(`AI PC connectionState=${aiPc.connectionState}`); + aiPc.oniceconnectionstatechange = () => log(`AI PC iceConnectionState=${aiPc.iceConnectionState}`); + + aiDc = aiPc.createDataChannel('chat'); + aiDc.onopen = () => { + setStatus('connected'); + log('AI data channel open', true); + aiDc.send(JSON.stringify({ type: 'client-ready' })); + }; + aiDc.onclose = () => setStatus('disconnected'); + aiDc.onmessage = (event) => { + try { + const msg = JSON.parse(event.data); + const content = msg.delta || msg.content; + if (msg.type === 'transcript') { + logUserTranscript(content); + } else if (msg.type === 'assistant-text-start') { + startAssistantLine(); + } else if (msg.type === 'assistant-text-delta') { + appendAssistantDelta(content); + } else if (msg.type === 'assistant-text-end') { + finishAssistantLine(); + } else { + log(`data channel message: ${msg.type}`); + } + } catch { + appendChatLine('DATA', event.data); + } + }; + + selectRemoteAudio('connect'); + selectBusinessSender('connect'); + + const audioTransceiver = isLive(selectedRemoteAudio) + ? aiPc.addTransceiver(selectedRemoteAudio, { direction: 'sendrecv' }) + : aiPc.addTransceiver('audio', { direction: 'sendrecv' }); + aiAudioSender = audioTransceiver.sender; + log(isLive(selectedRemoteAudio) ? `AI input starts with ${trackLabel(selectedRemoteAudio)}` : 'AI input waiting for selected audio', true); + + const shouldSendVideo = ENABLE_VIDEO_UPLINK && isLive(selectedRemoteVideo) && !selectedRemoteVideo.muted; + if (shouldSendVideo) { + const videoTransceiver = aiPc.addTransceiver(selectedRemoteVideo, { direction: 'sendonly' }); + aiVideoSender = videoTransceiver.sender; + log(`AI video starts with ${trackLabel(selectedRemoteVideo)}`); + } else { + aiVideoSender = null; + log('AI video disabled for this offer: no live native video track yet'); + } + + aiPc.ontrack = (event) => { + if (event.track.kind !== 'audio') return; + aiOutputTrackIds.add(event.track.id); + remoteAudio.delete(event.track.id); + aiAudioTrack = event.track; + log(`received AI audio ${trackLabel(aiAudioTrack)}`, true); + aiAudioTrack.onunmute = () => { + log(`AI audio unmuted ${trackLabel(aiAudioTrack)}`); + attachAiOutputToBusiness(); + }; + attachAiOutputToBusiness(); + }; + + aiPc.onicecandidate = (event) => { + if (aiWs?.readyState !== WebSocket.OPEN) return; + aiWs.send(JSON.stringify({ + type: 'ice-candidate', + payload: { + pc_id: id, + candidate: event.candidate ? { + candidate: event.candidate.candidate, + sdpMid: event.candidate.sdpMid, + sdpMLineIndex: event.candidate.sdpMLineIndex, + } : null, + }, + })); + }; + + setStatus('connecting'); + aiWs = new WebSocket(signalingUrl); + aiWs.onopen = async () => { + const offer = await aiPc.createOffer(); + await aiPc.setLocalDescription(offer); + aiWs.send(JSON.stringify({ + type: 'offer', + payload: { + pc_id: id, + sdp: aiPc.localDescription.sdp, + type: aiPc.localDescription.type, + assistant_id: assistantId, + vision_enabled: shouldSendVideo, + }, + })); + log('sent AI WebRTC offer', true); + }; + aiWs.onmessage = async (event) => { + try { + const msg = JSON.parse(event.data); + if (msg.type === 'answer') { + await aiPc.setRemoteDescription({ type: 'answer', sdp: msg.payload.sdp }); + log(`AI WebRTC answer applied; ${aiState()}`, true); + } else if (msg.type === 'ice-candidate' && msg.payload?.candidate) { + await aiPc.addIceCandidate(msg.payload.candidate).catch(() => {}); + } else if (msg.type === 'error') { + throw new Error(msg.payload?.message || 'server error'); + } + } catch (error) { + log(`signaling error: ${error.message || error}`); + } + }; + aiWs.onerror = () => { + setStatus('failed'); + log('AI signaling websocket error'); + }; + aiWs.onclose = () => setStatus('disconnected'); + + attachAudioToAi(); + attachVideoToAi(); + } + + function disconnectAi() { + try { aiDc?.close(); } catch {} + try { aiPc?.close(); } catch {} + try { aiWs?.close(); } catch {} + aiDc = null; + aiPc = null; + aiWs = null; + aiAudioSender = null; + aiVideoSender = null; + aiAudioTrack = null; + if (inputRecorder?.state === 'recording') stopInputRecording(); + setStatus('disconnected'); + } + + function setStatus(status) { + const el = document.getElementById('static-copilot-status'); + if (!el) return; + el.textContent = { + disconnected: '未连接', + connecting: '连接中', + connected: '已连接', + failed: '连接失败', + }[status] || status; + } + + function updateModeButton() { + const button = document.getElementById('static-copilot-toggle'); + if (!button) return; + if (isAiTakeover) { + button.textContent = '当前：AI 接管｜点击切回人工'; + button.style.background = '#8b5cf6'; + } else { + button.textContent = '当前：人工接管｜点击切换 AI'; + button.style.background = '#10b981'; + } + } + + window.__aiNativeStaticDump = function () { + return { + selectedInput: trackLabel(selectedRemoteAudio), + selectedVideo: trackLabel(selectedRemoteVideo), + aiVideoSenderTrack: trackLabel(aiVideoSender?.track), + selectedOutput: trackLabel(selectedBusinessSender?.track), + originalOutput: trackLabel(originalBusinessTrack), + aiAudio: trackLabel(aiAudioTrack), + aiAttached, + aiState: aiState(), + recording: inputRecorder?.state === 'recording' ? { + track: trackLabel(inputRecordingTrack), + seconds: ((Date.now() - inputRecordingStartedAt) / 1000).toFixed(1), + chunks: inputRecordingChunks.length, + } : null, + inputs: Array.from(remoteAudio.values()).map((item) => ({ + selected: item.track === selectedRemoteAudio, + id: item.track.id, + muted: item.track.muted, + live: item.track.readyState, + hasReceiver: Boolean(item.receiver), + bytesReceived: item.bytesReceived || 0, + packetsReceived: item.packetsReceived || 0, + bytesDelta: item.bytesDelta, + packetsDelta: item.packetsDelta, + reasons: Array.from(item.reasons), + })), + outputs: Array.from(businessSenders.values()).map((item) => ({ + selected: item.sender === selectedBusinessSender, + id: item.id, + track: trackLabel(item.sender.track || item.track), + originalTrack: trackLabel(item.originalTrack), + pcAddTrack: item.sources.has('pc-addTrack'), + isRemoteInput: isRemoteInputTrack(item.sender.track || item.track), + bytesSent: item.bytesSent, + packetsSent: item.packetsSent, + bytesDelta: item.bytesDelta, + packetsDelta: item.packetsDelta, + sources: Array.from(item.sources), + })), + }; + }; + + function renderPanel() { + if (!document.body || document.getElementById('static-copilot-panel')) return; + const panel = document.createElement('div'); + panel.id = 'static-copilot-panel'; + panel.style.cssText = [ + 'position:fixed', + 'top:20px', + 'left:20px', + 'width:340px', + 'z-index:999999', + 'box-sizing:border-box', + 'padding:16px', + 'border-radius:10px', + 'background:rgba(17,24,39,.96)', + 'color:white', + 'font-family:system-ui,-apple-system,BlinkMacSystemFont,sans-serif', + 'box-shadow:0 12px 32px rgba(0,0,0,.35)', + ].join(';'); + panel.innerHTML = ` +
+ AI 原生转接 + 未连接 +
+ + + + +
+
远端输入原生音轨
+
+
+
+
AI 输出业务上行
+
+
+ + +
+ 等待交互... +
+ `; + document.body.appendChild(panel); + + const toggle = document.getElementById('static-copilot-toggle'); + const recordButton = document.getElementById('static-copilot-record'); + const urlInput = document.getElementById('static-copilot-url'); + const assistantInput = document.getElementById('static-copilot-assistant'); + recordButton.addEventListener('click', toggleInputRecording); + toggle.addEventListener('click', async () => { + if (!isAiTakeover && !assistantInput.value.trim()) { + alert('请先填写助手 ID。'); + return; + } + isAiTakeover = !isAiTakeover; + updateModeButton(); + if (isAiTakeover) { + await connectAi(urlInput.value.trim(), assistantInput.value.trim()); + await attachAiOutputToBusiness(); + } else { + await restoreBusinessAudio(); + disconnectAi(); + } + }); + updateModeButton(); + updateRecordButton(); + } + + function renderPanelSoon() { + try { renderPanel(); } catch (error) { console.warn('[AI Native Static] panel error', error); } + } + + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', renderPanelSoon, { once: true }); + } else { + renderPanelSoon(); + } + requestAnimationFrame(renderVu); + setInterval(refreshStats, 250); + setTimeout(renderPanelSoon, 1500); + setTimeout(renderPanelSoon, 3000); + window.addEventListener('beforeunload', () => { + if (inputRecorder?.state === 'recording') stopInputRecording(); + }); +})();