Add AI Native Static Copilot script for enhanced media handling
- Introduce a new user script `copilot-native-static.js` that implements a native-track bridge with static input/output selection rules for WebRTC. - Implement functionality for managing remote audio and video tracks, including recording capabilities for AI input. - Enhance chat logging features to provide real-time feedback on AI interactions and recording status. - Integrate mechanisms for handling media streams and improve overall media management in the WebRTC environment.
This commit is contained in:
Xin Wang
876
scripts/copilot-native-static.js
Normal file
876
scripts/copilot-native-static.js
Normal file
@@ -0,0 +1,876 @@
|
||||
// ==UserScript==
|
||||
// @name 12345 AI Native Static Copilot
|
||||
// @namespace http://tampermonkey.net/
|
||||
// @version 1.0
|
||||
// @description Native-track bridge with static input/output selection rules.
|
||||
// @author You
|
||||
// @match *://yhy-yw.22sj.cn:*/*
|
||||
// @grant none
|
||||
// @run-at document-start
|
||||
// ==/UserScript==
|
||||
|
||||
(function () {
|
||||
'use strict';
|
||||
|
||||
const DEFAULT_SIGNALING_URL = 'ws://182.92.86.220:8000/ws/voice';
|
||||
|
||||
const originalAddEventListener = RTCPeerConnection.prototype.addEventListener;
|
||||
const originalAddTrack = RTCPeerConnection.prototype.addTrack;
|
||||
const originalAddTransceiver = RTCPeerConnection.prototype.addTransceiver;
|
||||
const originalSetRemoteDescription = RTCPeerConnection.prototype.setRemoteDescription;
|
||||
const mediaSrcObjectDescriptor = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, 'srcObject');
|
||||
|
||||
const remoteAudio = new Map();
|
||||
const remoteVideo = new Map();
|
||||
const businessSenders = new Map();
|
||||
const aiOutputTrackIds = new Set();
|
||||
const inboundMemory = new Map();
|
||||
const outboundMemory = new Map();
|
||||
const internalPcs = new WeakSet();
|
||||
const ENABLE_VIDEO_UPLINK = true;
|
||||
|
||||
let selectedRemoteAudio = null;
|
||||
let selectedRemoteVideo = null;
|
||||
let selectedBusinessSender = null;
|
||||
let originalBusinessTrack = null;
|
||||
let aiAudioTrack = null;
|
||||
|
||||
let aiPc = null;
|
||||
let aiWs = null;
|
||||
let aiDc = null;
|
||||
let aiAudioSender = null;
|
||||
let aiVideoSender = null;
|
||||
let isAiTakeover = false;
|
||||
let aiAttached = false;
|
||||
let inputVu = 0;
|
||||
let outputVu = 0;
|
||||
let inputHealthTimer = null;
|
||||
let outputHealthTimer = null;
|
||||
let assistantLine = null;
|
||||
let assistantBuffer = '';
|
||||
let inputRecorder = null;
|
||||
let inputRecordingChunks = [];
|
||||
let inputRecordingTrack = null;
|
||||
let inputRecordingStartedAt = 0;
|
||||
|
||||
function log(message) {
|
||||
console.log(`[AI Native Static] ${message}`);
|
||||
}
|
||||
|
||||
function appendChatLine(role, text, important = false) {
|
||||
const box = document.getElementById('static-copilot-log');
|
||||
if (!box) return;
|
||||
if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = '';
|
||||
const row = document.createElement('div');
|
||||
row.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] ${role}: ${text}`;
|
||||
row.style.marginBottom = '4px';
|
||||
if (important) row.style.color = '#fcd34d';
|
||||
box.appendChild(row);
|
||||
while (box.children.length > 120) box.removeChild(box.firstChild);
|
||||
box.scrollTop = box.scrollHeight;
|
||||
return row;
|
||||
}
|
||||
|
||||
function startAssistantLine() {
|
||||
assistantBuffer = '';
|
||||
assistantLine = null;
|
||||
}
|
||||
|
||||
function appendAssistantDelta(delta) {
|
||||
if (!delta) return;
|
||||
const box = document.getElementById('static-copilot-log');
|
||||
if (!box) {
|
||||
console.log(`[AI Native Static] AI: ${delta}`);
|
||||
return;
|
||||
}
|
||||
if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = '';
|
||||
if (!assistantLine) assistantLine = appendChatLine('AI', '');
|
||||
assistantBuffer += delta;
|
||||
assistantLine.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] AI: ${assistantBuffer}`;
|
||||
box.scrollTop = box.scrollHeight;
|
||||
}
|
||||
|
||||
function finishAssistantLine() {
|
||||
assistantLine = null;
|
||||
assistantBuffer = '';
|
||||
}
|
||||
|
||||
function logUserTranscript(text) {
|
||||
if (!text?.trim()) return;
|
||||
finishAssistantLine();
|
||||
appendChatLine('用户', text.trim(), true);
|
||||
}
|
||||
|
||||
function recorderMimeType() {
|
||||
if (window.MediaRecorder?.isTypeSupported?.('audio/webm;codecs=opus')) return 'audio/webm;codecs=opus';
|
||||
if (window.MediaRecorder?.isTypeSupported?.('audio/webm')) return 'audio/webm';
|
||||
return '';
|
||||
}
|
||||
|
||||
function recordingFileName(track) {
|
||||
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track';
|
||||
return `ai-input-${trackPart}-${stamp}.webm`;
|
||||
}
|
||||
|
||||
async function startInputRecording() {
|
||||
if (inputRecorder?.state === 'recording') return;
|
||||
selectRemoteAudio('record');
|
||||
if (!isLive(selectedRemoteAudio)) {
|
||||
appendChatLine('录音', '没有可录制的远端输入音轨', true);
|
||||
return;
|
||||
}
|
||||
if (!window.MediaRecorder) {
|
||||
appendChatLine('录音', '当前浏览器不支持 MediaRecorder', true);
|
||||
return;
|
||||
}
|
||||
|
||||
inputRecordingChunks = [];
|
||||
inputRecordingTrack = selectedRemoteAudio;
|
||||
inputRecordingStartedAt = Date.now();
|
||||
const stream = new MediaStream([selectedRemoteAudio]);
|
||||
const mimeType = recorderMimeType();
|
||||
inputRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined);
|
||||
inputRecorder.ondataavailable = (event) => {
|
||||
if (event.data?.size > 0) inputRecordingChunks.push(event.data);
|
||||
};
|
||||
inputRecorder.onstop = () => {
|
||||
const blob = new Blob(inputRecordingChunks, { type: inputRecorder?.mimeType || 'audio/webm' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
const link = document.createElement('a');
|
||||
link.href = url;
|
||||
link.download = recordingFileName(inputRecordingTrack);
|
||||
document.body.appendChild(link);
|
||||
link.click();
|
||||
link.remove();
|
||||
setTimeout(() => URL.revokeObjectURL(url), 30000);
|
||||
|
||||
const seconds = ((Date.now() - inputRecordingStartedAt) / 1000).toFixed(1);
|
||||
appendChatLine('录音', `已保存 ${link.download}（${seconds}s，${(blob.size / 1024).toFixed(1)} KB）`, true);
|
||||
inputRecorder = null;
|
||||
inputRecordingChunks = [];
|
||||
inputRecordingTrack = null;
|
||||
updateRecordButton();
|
||||
};
|
||||
inputRecorder.start(250);
|
||||
appendChatLine('录音', `开始录制 AI 输入：${trackLabel(inputRecordingTrack)}`, true);
|
||||
updateRecordButton();
|
||||
}
|
||||
|
||||
function stopInputRecording() {
|
||||
if (inputRecorder?.state === 'recording') inputRecorder.stop();
|
||||
}
|
||||
|
||||
function toggleInputRecording() {
|
||||
if (inputRecorder?.state === 'recording') stopInputRecording();
|
||||
else startInputRecording();
|
||||
}
|
||||
|
||||
function updateRecordButton() {
|
||||
const button = document.getElementById('static-copilot-record');
|
||||
if (!button) return;
|
||||
const recording = inputRecorder?.state === 'recording';
|
||||
button.textContent = recording ? '停止录制 AI 输入' : '录制 AI 输入';
|
||||
button.style.background = recording ? '#dc2626' : '#2563eb';
|
||||
}
|
||||
|
||||
function pcId() {
|
||||
const bytes = new Uint8Array(16);
|
||||
crypto.getRandomValues(bytes);
|
||||
return 'PC-' + Array.from(bytes).map((b) => b.toString(16).padStart(2, '0')).join('');
|
||||
}
|
||||
|
||||
function isInternalPc(pc) {
|
||||
return Boolean(pc && (pc.__aiNativeStaticInternal || internalPcs.has(pc) || pc === aiPc));
|
||||
}
|
||||
|
||||
function isLive(track) {
|
||||
return track && track.readyState === 'live';
|
||||
}
|
||||
|
||||
function trackLabel(track) {
|
||||
if (!track) return 'none';
|
||||
return `${track.id}${track.muted ? ' muted' : ''}`;
|
||||
}
|
||||
|
||||
function markInternalPc(pc) {
|
||||
if (!pc) return;
|
||||
pc.__aiNativeStaticInternal = true;
|
||||
internalPcs.add(pc);
|
||||
}
|
||||
|
||||
function rememberRemoteReceiver(receiver, reason) {
|
||||
const track = receiver?.track;
|
||||
if (!isLive(track) || isInternalPc(receiver?.transport)) return;
|
||||
|
||||
if (track.kind === 'audio' && !aiOutputTrackIds.has(track.id)) {
|
||||
let item = remoteAudio.get(track.id);
|
||||
if (!item) {
|
||||
item = { track, receiver, reasons: new Set(), bytesDelta: 0, packetsDelta: 0 };
|
||||
remoteAudio.set(track.id, item);
|
||||
log(`audio candidate ${trackLabel(track)} (${reason})`, true);
|
||||
}
|
||||
item.track = track;
|
||||
item.receiver = receiver || item.receiver;
|
||||
item.reasons.add(reason);
|
||||
selectRemoteAudio(reason);
|
||||
}
|
||||
|
||||
if (track.kind === 'video') {
|
||||
let item = remoteVideo.get(track.id);
|
||||
if (!item) {
|
||||
item = { track, receiver, reasons: new Set() };
|
||||
remoteVideo.set(track.id, item);
|
||||
log(`video candidate ${trackLabel(track)} (${reason})`, true);
|
||||
}
|
||||
item.track = track;
|
||||
item.receiver = receiver || item.receiver;
|
||||
item.reasons.add(reason);
|
||||
if (!selectedRemoteVideo || selectedRemoteVideo.readyState !== 'live') {
|
||||
selectedRemoteVideo = track;
|
||||
attachVideoToAi();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function rememberRemoteStream(stream, reason) {
|
||||
if (!stream || !(stream instanceof MediaStream)) return;
|
||||
for (const track of stream.getAudioTracks()) {
|
||||
if (aiOutputTrackIds.has(track.id)) continue;
|
||||
let item = remoteAudio.get(track.id);
|
||||
if (!item) {
|
||||
item = { track, receiver: null, reasons: new Set(), bytesDelta: 0, packetsDelta: 0 };
|
||||
remoteAudio.set(track.id, item);
|
||||
log(`audio stream candidate ${trackLabel(track)} (${reason})`, true);
|
||||
}
|
||||
item.track = track;
|
||||
item.reasons.add(reason);
|
||||
}
|
||||
for (const track of stream.getVideoTracks()) {
|
||||
let item = remoteVideo.get(track.id);
|
||||
if (!item) {
|
||||
item = { track, receiver: null, reasons: new Set() };
|
||||
remoteVideo.set(track.id, item);
|
||||
log(`video stream candidate ${trackLabel(track)} (${reason})`, true);
|
||||
}
|
||||
item.track = track;
|
||||
item.reasons.add(reason);
|
||||
if (!selectedRemoteVideo || selectedRemoteVideo.readyState !== 'live') selectedRemoteVideo = track;
|
||||
}
|
||||
selectRemoteAudio(reason);
|
||||
attachVideoToAi();
|
||||
}
|
||||
|
||||
function selectRemoteAudio(reason) {
|
||||
if (isLive(selectedRemoteAudio) && !selectedRemoteAudio.muted) return selectedRemoteAudio;
|
||||
|
||||
const item = Array.from(remoteAudio.values())
|
||||
.find((candidate) => isLive(candidate.track) && !candidate.track.muted && candidate.receiver);
|
||||
if (!item) return null;
|
||||
|
||||
selectedRemoteAudio = item.track;
|
||||
log(`selected input audio (${reason}): ${trackLabel(selectedRemoteAudio)}`, true);
|
||||
attachAudioToAi();
|
||||
return selectedRemoteAudio;
|
||||
}
|
||||
|
||||
function rememberBusinessSender(sender, track, reason) {
|
||||
if (!sender) return;
|
||||
const senderTrack = track || sender.track;
|
||||
if (senderTrack && senderTrack.kind !== 'audio') return;
|
||||
|
||||
const id = sender.__aiNativeStaticSenderId || `sender-${businessSenders.size + 1}`;
|
||||
sender.__aiNativeStaticSenderId = id;
|
||||
|
||||
let item = businessSenders.get(id);
|
||||
if (!item) {
|
||||
item = { id, sender, track: senderTrack || null, originalTrack: senderTrack || null, sources: new Set(), bytesDelta: 0, packetsDelta: 0 };
|
||||
businessSenders.set(id, item);
|
||||
log(`business sender candidate ${id} (${reason}): ${trackLabel(senderTrack)}`, true);
|
||||
}
|
||||
item.sender = sender;
|
||||
item.track = sender.track || senderTrack || item.track;
|
||||
if (!isAiTakeover && senderTrack && senderTrack !== aiAudioTrack) item.originalTrack = senderTrack;
|
||||
item.sources.add(reason);
|
||||
wrapBusinessSender(sender);
|
||||
selectBusinessSender(reason);
|
||||
}
|
||||
|
||||
function wrapBusinessSender(sender) {
|
||||
if (!sender || sender.__aiNativeStaticWrapped) return;
|
||||
const originalReplaceTrack = sender.replaceTrack?.bind(sender);
|
||||
if (!originalReplaceTrack) return;
|
||||
|
||||
sender.replaceTrack = async function (track) {
|
||||
const result = await originalReplaceTrack(track);
|
||||
rememberBusinessSender(sender, track, 'replaceTrack');
|
||||
return result;
|
||||
};
|
||||
sender.__aiNativeStaticWrapped = true;
|
||||
}
|
||||
|
||||
function isRemoteInputTrack(track) {
|
||||
return Boolean(track && remoteAudio.has(track.id) && !aiOutputTrackIds.has(track.id));
|
||||
}
|
||||
|
||||
function selectBusinessSender(reason) {
|
||||
if (selectedBusinessSender && !isRemoteInputTrack(selectedBusinessSender.track)) return selectedBusinessSender;
|
||||
|
||||
const item = Array.from(businessSenders.values()).find((candidate) => {
|
||||
const track = candidate.sender.track || candidate.track || candidate.originalTrack;
|
||||
return candidate.sources.has('pc-addTrack') && !isRemoteInputTrack(track);
|
||||
});
|
||||
if (!item) return null;
|
||||
|
||||
selectedBusinessSender = item.sender;
|
||||
originalBusinessTrack = item.originalTrack || item.sender.track || originalBusinessTrack;
|
||||
log(`selected output sender (${reason}): ${item.id}, track=${trackLabel(item.sender.track)}`, true);
|
||||
attachAiOutputToBusiness();
|
||||
return selectedBusinessSender;
|
||||
}
|
||||
|
||||
function setVu(id, level) {
|
||||
const el = document.getElementById(id);
|
||||
if (el) el.style.width = `${Math.min(100, level * 2500)}%`;
|
||||
}
|
||||
|
||||
function renderVu() {
|
||||
inputVu *= 0.88;
|
||||
outputVu *= 0.88;
|
||||
setVu('static-copilot-input-vu', inputVu);
|
||||
setVu('static-copilot-output-vu', outputVu);
|
||||
requestAnimationFrame(renderVu);
|
||||
}
|
||||
|
||||
async function refreshStats() {
|
||||
await refreshInputStats();
|
||||
await refreshOutputStats();
|
||||
}
|
||||
|
||||
async function refreshInputStats() {
|
||||
for (const item of remoteAudio.values()) {
|
||||
if (!item.receiver?.getStats) continue;
|
||||
const report = await item.receiver.getStats();
|
||||
report.forEach((stat) => {
|
||||
const kind = stat.kind || stat.mediaType;
|
||||
if (stat.type !== 'inbound-rtp' || kind !== 'audio') return;
|
||||
const prev = inboundMemory.get(stat.id);
|
||||
const bytes = stat.bytesReceived || 0;
|
||||
const packets = stat.packetsReceived || 0;
|
||||
item.bytesReceived = bytes;
|
||||
item.packetsReceived = packets;
|
||||
item.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0;
|
||||
item.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0;
|
||||
inboundMemory.set(stat.id, { bytes, packets });
|
||||
if (item.track === selectedRemoteAudio) {
|
||||
inputVu = Math.max(inputVu, Math.min(0.12, item.bytesDelta / 12000), Math.min(0.04, item.packetsDelta / 500));
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshOutputStats() {
|
||||
for (const item of businessSenders.values()) {
|
||||
if (!item.sender?.getStats) continue;
|
||||
item.track = item.sender.track || item.track;
|
||||
const report = await item.sender.getStats();
|
||||
report.forEach((stat) => {
|
||||
const kind = stat.kind || stat.mediaType;
|
||||
if (stat.type !== 'outbound-rtp' || kind !== 'audio') return;
|
||||
const key = `${item.id}:${stat.id}`;
|
||||
const prev = outboundMemory.get(key);
|
||||
const bytes = stat.bytesSent || 0;
|
||||
const packets = stat.packetsSent || 0;
|
||||
item.bytesSent = bytes;
|
||||
item.packetsSent = packets;
|
||||
item.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0;
|
||||
item.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0;
|
||||
outboundMemory.set(key, { bytes, packets });
|
||||
if (item.sender === selectedBusinessSender) {
|
||||
outputVu = Math.max(outputVu, Math.min(0.12, item.bytesDelta / 12000), Math.min(0.04, item.packetsDelta / 500));
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function outboundBytes(sender) {
|
||||
if (!sender?.getStats) return null;
|
||||
let total = 0;
|
||||
const report = await sender.getStats();
|
||||
report.forEach((stat) => {
|
||||
const kind = stat.kind || stat.mediaType;
|
||||
if (stat.type === 'outbound-rtp' && kind === 'audio') total += stat.bytesSent || 0;
|
||||
});
|
||||
return total;
|
||||
}
|
||||
|
||||
function aiState() {
|
||||
if (!aiPc) return 'pc=null';
|
||||
return `conn=${aiPc.connectionState},ice=${aiPc.iceConnectionState},signaling=${aiPc.signalingState},remote=${aiPc.remoteDescription?.type || 'none'}`;
|
||||
}
|
||||
|
||||
function aiReady() {
|
||||
return Boolean(aiPc?.remoteDescription && (
|
||||
aiPc.connectionState === 'connected'
|
||||
|| aiPc.iceConnectionState === 'connected'
|
||||
|| aiPc.iceConnectionState === 'completed'
|
||||
));
|
||||
}
|
||||
|
||||
function waitForAiReady(timeoutMs = 10000) {
|
||||
if (aiReady()) return Promise.resolve(true);
|
||||
return new Promise((resolve) => {
|
||||
const startedAt = Date.now();
|
||||
const timer = setInterval(() => {
|
||||
if (aiReady()) {
|
||||
clearInterval(timer);
|
||||
resolve(true);
|
||||
} else if (Date.now() - startedAt >= timeoutMs) {
|
||||
clearInterval(timer);
|
||||
resolve(false);
|
||||
}
|
||||
}, 250);
|
||||
});
|
||||
}
|
||||
|
||||
function checkAiInput(track) {
|
||||
if (inputHealthTimer) clearTimeout(inputHealthTimer);
|
||||
inputHealthTimer = setTimeout(async () => {
|
||||
if (!aiAudioSender || aiAudioSender.track !== track) return;
|
||||
await waitForAiReady();
|
||||
const before = await outboundBytes(aiAudioSender);
|
||||
await new Promise((resolve) => setTimeout(resolve, 2500));
|
||||
const after = await outboundBytes(aiAudioSender);
|
||||
log(`AI input packet check: before=${before}, after=${after}, ${aiState()}`, true);
|
||||
}, 500);
|
||||
}
|
||||
|
||||
function checkAiOutput(track) {
|
||||
if (outputHealthTimer) clearTimeout(outputHealthTimer);
|
||||
outputHealthTimer = setTimeout(async () => {
|
||||
if (!selectedBusinessSender || selectedBusinessSender.track !== track) return;
|
||||
const before = await outboundBytes(selectedBusinessSender);
|
||||
await new Promise((resolve) => setTimeout(resolve, 2500));
|
||||
const after = await outboundBytes(selectedBusinessSender);
|
||||
log(`AI output packet check: before=${before}, after=${after}, sender=${trackLabel(selectedBusinessSender.track)}`, true);
|
||||
}, 800);
|
||||
}
|
||||
|
||||
async function attachAudioToAi() {
|
||||
if (!aiAudioSender || !isLive(selectedRemoteAudio)) return false;
|
||||
if (aiAudioSender.track === selectedRemoteAudio) return true;
|
||||
await aiAudioSender.replaceTrack(selectedRemoteAudio);
|
||||
log(`AI input uses ${trackLabel(selectedRemoteAudio)}`, true);
|
||||
checkAiInput(selectedRemoteAudio);
|
||||
return true;
|
||||
}
|
||||
|
||||
async function attachVideoToAi() {
|
||||
if (!aiVideoSender || !isLive(selectedRemoteVideo)) return false;
|
||||
if (aiVideoSender.track === selectedRemoteVideo) return true;
|
||||
await aiVideoSender.replaceTrack(selectedRemoteVideo);
|
||||
log(`AI video uses ${trackLabel(selectedRemoteVideo)}`, true);
|
||||
return true;
|
||||
}
|
||||
|
||||
async function attachAiOutputToBusiness() {
|
||||
selectBusinessSender('before-output');
|
||||
if (!isAiTakeover || !selectedBusinessSender || !isLive(aiAudioTrack)) {
|
||||
log(`AI output waiting: takeover=${isAiTakeover}, sender=${Boolean(selectedBusinessSender)}, aiTrack=${trackLabel(aiAudioTrack)}`);
|
||||
return false;
|
||||
}
|
||||
if (selectedBusinessSender.track === aiAudioTrack) return true;
|
||||
await selectedBusinessSender.replaceTrack(aiAudioTrack);
|
||||
aiAttached = true;
|
||||
log(`business sender uses AI audio ${trackLabel(aiAudioTrack)}`, true);
|
||||
checkAiOutput(aiAudioTrack);
|
||||
return true;
|
||||
}
|
||||
|
||||
async function restoreBusinessAudio() {
|
||||
if (!selectedBusinessSender || !aiAttached) return;
|
||||
await selectedBusinessSender.replaceTrack(originalBusinessTrack || null);
|
||||
aiAttached = false;
|
||||
log(`business sender restored ${trackLabel(originalBusinessTrack)}`, true);
|
||||
}
|
||||
|
||||
function wrapTrackListener(listener) {
|
||||
if (!listener || listener.__aiNativeStaticWrapped) return listener;
|
||||
const wrapped = function (event) {
|
||||
const ownerPc = this instanceof RTCPeerConnection ? this : event?.currentTarget;
|
||||
if (!isInternalPc(ownerPc) && event?.track) {
|
||||
rememberRemoteReceiver(event.receiver, 'pc-track');
|
||||
for (const stream of event.streams || []) rememberRemoteStream(stream, 'pc-track-stream');
|
||||
}
|
||||
if (typeof listener === 'function') return listener.apply(this, arguments);
|
||||
if (listener?.handleEvent) return listener.handleEvent(event);
|
||||
return undefined;
|
||||
};
|
||||
wrapped.__aiNativeStaticWrapped = true;
|
||||
return wrapped;
|
||||
}
|
||||
|
||||
RTCPeerConnection.prototype.addEventListener = function (type, listener, options) {
|
||||
if (type === 'track') return originalAddEventListener.call(this, type, wrapTrackListener(listener), options);
|
||||
return originalAddEventListener.call(this, type, listener, options);
|
||||
};
|
||||
|
||||
try {
|
||||
const descriptor = Object.getOwnPropertyDescriptor(RTCPeerConnection.prototype, 'ontrack');
|
||||
Object.defineProperty(RTCPeerConnection.prototype, 'ontrack', {
|
||||
configurable: true,
|
||||
enumerable: true,
|
||||
get() {
|
||||
return descriptor?.get ? descriptor.get.call(this) : this.__aiNativeStaticOnTrack || null;
|
||||
},
|
||||
set(listener) {
|
||||
const wrapped = wrapTrackListener(listener);
|
||||
this.__aiNativeStaticOnTrack = wrapped;
|
||||
if (descriptor?.set) descriptor.set.call(this, wrapped);
|
||||
else this.addEventListener('track', wrapped);
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.warn('[AI Native Static] failed to wrap ontrack', error);
|
||||
}
|
||||
|
||||
RTCPeerConnection.prototype.addTrack = function (track, ...streams) {
|
||||
const sender = originalAddTrack.call(this, track, ...streams);
|
||||
if (!isInternalPc(this) && track?.kind === 'audio') rememberBusinessSender(sender, track, 'pc-addTrack');
|
||||
return sender;
|
||||
};
|
||||
|
||||
if (originalAddTransceiver) {
|
||||
RTCPeerConnection.prototype.addTransceiver = function (trackOrKind, init) {
|
||||
const transceiver = originalAddTransceiver.call(this, trackOrKind, init);
|
||||
if (!isInternalPc(this)) {
|
||||
const kind = trackOrKind instanceof MediaStreamTrack ? trackOrKind.kind : trackOrKind;
|
||||
if (kind === 'audio') {
|
||||
const track = trackOrKind instanceof MediaStreamTrack ? trackOrKind : transceiver.sender?.track;
|
||||
rememberBusinessSender(transceiver.sender, track, 'pc-addTransceiver');
|
||||
}
|
||||
}
|
||||
return transceiver;
|
||||
};
|
||||
}
|
||||
|
||||
RTCPeerConnection.prototype.setRemoteDescription = function () {
|
||||
const result = originalSetRemoteDescription.apply(this, arguments);
|
||||
if (!isInternalPc(this)) {
|
||||
Promise.resolve(result).then(() => {
|
||||
for (const receiver of this.getReceivers?.() || []) rememberRemoteReceiver(receiver, 'receiver-after-srd');
|
||||
for (const sender of this.getSenders?.() || []) rememberBusinessSender(sender, sender.track, 'sender-after-srd');
|
||||
}).catch(() => {});
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
if (mediaSrcObjectDescriptor?.set) {
|
||||
Object.defineProperty(HTMLMediaElement.prototype, 'srcObject', {
|
||||
configurable: true,
|
||||
enumerable: mediaSrcObjectDescriptor.enumerable,
|
||||
get() {
|
||||
return mediaSrcObjectDescriptor.get.call(this);
|
||||
},
|
||||
set(stream) {
|
||||
if (!this.__aiNativeStaticElement) rememberRemoteStream(stream, `${this.tagName.toLowerCase()}-srcObject`);
|
||||
return mediaSrcObjectDescriptor.set.call(this, stream);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
setInterval(() => {
|
||||
for (const element of document.querySelectorAll('audio, video')) {
|
||||
if (!element.__aiNativeStaticElement) rememberRemoteStream(element.srcObject, 'media-scan');
|
||||
}
|
||||
}, 1000);
|
||||
|
||||
async function connectAi(signalingUrl, assistantId) {
|
||||
disconnectAi();
|
||||
|
||||
const id = pcId();
|
||||
aiPc = new RTCPeerConnection({ iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] });
|
||||
markInternalPc(aiPc);
|
||||
aiPc.onconnectionstatechange = () => log(`AI PC connectionState=${aiPc.connectionState}`);
|
||||
aiPc.oniceconnectionstatechange = () => log(`AI PC iceConnectionState=${aiPc.iceConnectionState}`);
|
||||
|
||||
aiDc = aiPc.createDataChannel('chat');
|
||||
aiDc.onopen = () => {
|
||||
setStatus('connected');
|
||||
log('AI data channel open', true);
|
||||
aiDc.send(JSON.stringify({ type: 'client-ready' }));
|
||||
};
|
||||
aiDc.onclose = () => setStatus('disconnected');
|
||||
aiDc.onmessage = (event) => {
|
||||
try {
|
||||
const msg = JSON.parse(event.data);
|
||||
const content = msg.delta || msg.content;
|
||||
if (msg.type === 'transcript') {
|
||||
logUserTranscript(content);
|
||||
} else if (msg.type === 'assistant-text-start') {
|
||||
startAssistantLine();
|
||||
} else if (msg.type === 'assistant-text-delta') {
|
||||
appendAssistantDelta(content);
|
||||
} else if (msg.type === 'assistant-text-end') {
|
||||
finishAssistantLine();
|
||||
} else {
|
||||
log(`data channel message: ${msg.type}`);
|
||||
}
|
||||
} catch {
|
||||
appendChatLine('DATA', event.data);
|
||||
}
|
||||
};
|
||||
|
||||
selectRemoteAudio('connect');
|
||||
selectBusinessSender('connect');
|
||||
|
||||
const audioTransceiver = isLive(selectedRemoteAudio)
|
||||
? aiPc.addTransceiver(selectedRemoteAudio, { direction: 'sendrecv' })
|
||||
: aiPc.addTransceiver('audio', { direction: 'sendrecv' });
|
||||
aiAudioSender = audioTransceiver.sender;
|
||||
log(isLive(selectedRemoteAudio) ? `AI input starts with ${trackLabel(selectedRemoteAudio)}` : 'AI input waiting for selected audio', true);
|
||||
|
||||
const shouldSendVideo = ENABLE_VIDEO_UPLINK && isLive(selectedRemoteVideo) && !selectedRemoteVideo.muted;
|
||||
if (shouldSendVideo) {
|
||||
const videoTransceiver = aiPc.addTransceiver(selectedRemoteVideo, { direction: 'sendonly' });
|
||||
aiVideoSender = videoTransceiver.sender;
|
||||
log(`AI video starts with ${trackLabel(selectedRemoteVideo)}`);
|
||||
} else {
|
||||
aiVideoSender = null;
|
||||
log('AI video disabled for this offer: no live native video track yet');
|
||||
}
|
||||
|
||||
aiPc.ontrack = (event) => {
|
||||
if (event.track.kind !== 'audio') return;
|
||||
aiOutputTrackIds.add(event.track.id);
|
||||
remoteAudio.delete(event.track.id);
|
||||
aiAudioTrack = event.track;
|
||||
log(`received AI audio ${trackLabel(aiAudioTrack)}`, true);
|
||||
aiAudioTrack.onunmute = () => {
|
||||
log(`AI audio unmuted ${trackLabel(aiAudioTrack)}`);
|
||||
attachAiOutputToBusiness();
|
||||
};
|
||||
attachAiOutputToBusiness();
|
||||
};
|
||||
|
||||
aiPc.onicecandidate = (event) => {
|
||||
if (aiWs?.readyState !== WebSocket.OPEN) return;
|
||||
aiWs.send(JSON.stringify({
|
||||
type: 'ice-candidate',
|
||||
payload: {
|
||||
pc_id: id,
|
||||
candidate: event.candidate ? {
|
||||
candidate: event.candidate.candidate,
|
||||
sdpMid: event.candidate.sdpMid,
|
||||
sdpMLineIndex: event.candidate.sdpMLineIndex,
|
||||
} : null,
|
||||
},
|
||||
}));
|
||||
};
|
||||
|
||||
setStatus('connecting');
|
||||
aiWs = new WebSocket(signalingUrl);
|
||||
aiWs.onopen = async () => {
|
||||
const offer = await aiPc.createOffer();
|
||||
await aiPc.setLocalDescription(offer);
|
||||
aiWs.send(JSON.stringify({
|
||||
type: 'offer',
|
||||
payload: {
|
||||
pc_id: id,
|
||||
sdp: aiPc.localDescription.sdp,
|
||||
type: aiPc.localDescription.type,
|
||||
assistant_id: assistantId,
|
||||
vision_enabled: shouldSendVideo,
|
||||
},
|
||||
}));
|
||||
log('sent AI WebRTC offer', true);
|
||||
};
|
||||
aiWs.onmessage = async (event) => {
|
||||
try {
|
||||
const msg = JSON.parse(event.data);
|
||||
if (msg.type === 'answer') {
|
||||
await aiPc.setRemoteDescription({ type: 'answer', sdp: msg.payload.sdp });
|
||||
log(`AI WebRTC answer applied; ${aiState()}`, true);
|
||||
} else if (msg.type === 'ice-candidate' && msg.payload?.candidate) {
|
||||
await aiPc.addIceCandidate(msg.payload.candidate).catch(() => {});
|
||||
} else if (msg.type === 'error') {
|
||||
throw new Error(msg.payload?.message || 'server error');
|
||||
}
|
||||
} catch (error) {
|
||||
log(`signaling error: ${error.message || error}`);
|
||||
}
|
||||
};
|
||||
aiWs.onerror = () => {
|
||||
setStatus('failed');
|
||||
log('AI signaling websocket error');
|
||||
};
|
||||
aiWs.onclose = () => setStatus('disconnected');
|
||||
|
||||
attachAudioToAi();
|
||||
attachVideoToAi();
|
||||
}
|
||||
|
||||
function disconnectAi() {
|
||||
try { aiDc?.close(); } catch {}
|
||||
try { aiPc?.close(); } catch {}
|
||||
try { aiWs?.close(); } catch {}
|
||||
aiDc = null;
|
||||
aiPc = null;
|
||||
aiWs = null;
|
||||
aiAudioSender = null;
|
||||
aiVideoSender = null;
|
||||
aiAudioTrack = null;
|
||||
if (inputRecorder?.state === 'recording') stopInputRecording();
|
||||
setStatus('disconnected');
|
||||
}
|
||||
|
||||
function setStatus(status) {
|
||||
const el = document.getElementById('static-copilot-status');
|
||||
if (!el) return;
|
||||
el.textContent = {
|
||||
disconnected: '未连接',
|
||||
connecting: '连接中',
|
||||
connected: '已连接',
|
||||
failed: '连接失败',
|
||||
}[status] || status;
|
||||
}
|
||||
|
||||
function updateModeButton() {
|
||||
const button = document.getElementById('static-copilot-toggle');
|
||||
if (!button) return;
|
||||
if (isAiTakeover) {
|
||||
button.textContent = '当前：AI 接管｜点击切回人工';
|
||||
button.style.background = '#8b5cf6';
|
||||
} else {
|
||||
button.textContent = '当前：人工接管｜点击切换 AI';
|
||||
button.style.background = '#10b981';
|
||||
}
|
||||
}
|
||||
|
||||
window.__aiNativeStaticDump = function () {
|
||||
return {
|
||||
selectedInput: trackLabel(selectedRemoteAudio),
|
||||
selectedVideo: trackLabel(selectedRemoteVideo),
|
||||
aiVideoSenderTrack: trackLabel(aiVideoSender?.track),
|
||||
selectedOutput: trackLabel(selectedBusinessSender?.track),
|
||||
originalOutput: trackLabel(originalBusinessTrack),
|
||||
aiAudio: trackLabel(aiAudioTrack),
|
||||
aiAttached,
|
||||
aiState: aiState(),
|
||||
recording: inputRecorder?.state === 'recording' ? {
|
||||
track: trackLabel(inputRecordingTrack),
|
||||
seconds: ((Date.now() - inputRecordingStartedAt) / 1000).toFixed(1),
|
||||
chunks: inputRecordingChunks.length,
|
||||
} : null,
|
||||
inputs: Array.from(remoteAudio.values()).map((item) => ({
|
||||
selected: item.track === selectedRemoteAudio,
|
||||
id: item.track.id,
|
||||
muted: item.track.muted,
|
||||
live: item.track.readyState,
|
||||
hasReceiver: Boolean(item.receiver),
|
||||
bytesReceived: item.bytesReceived || 0,
|
||||
packetsReceived: item.packetsReceived || 0,
|
||||
bytesDelta: item.bytesDelta,
|
||||
packetsDelta: item.packetsDelta,
|
||||
reasons: Array.from(item.reasons),
|
||||
})),
|
||||
outputs: Array.from(businessSenders.values()).map((item) => ({
|
||||
selected: item.sender === selectedBusinessSender,
|
||||
id: item.id,
|
||||
track: trackLabel(item.sender.track || item.track),
|
||||
originalTrack: trackLabel(item.originalTrack),
|
||||
pcAddTrack: item.sources.has('pc-addTrack'),
|
||||
isRemoteInput: isRemoteInputTrack(item.sender.track || item.track),
|
||||
bytesSent: item.bytesSent,
|
||||
packetsSent: item.packetsSent,
|
||||
bytesDelta: item.bytesDelta,
|
||||
packetsDelta: item.packetsDelta,
|
||||
sources: Array.from(item.sources),
|
||||
})),
|
||||
};
|
||||
};
|
||||
|
||||
function renderPanel() {
|
||||
if (!document.body || document.getElementById('static-copilot-panel')) return;
|
||||
const panel = document.createElement('div');
|
||||
panel.id = 'static-copilot-panel';
|
||||
panel.style.cssText = [
|
||||
'position:fixed',
|
||||
'top:20px',
|
||||
'left:20px',
|
||||
'width:340px',
|
||||
'z-index:999999',
|
||||
'box-sizing:border-box',
|
||||
'padding:16px',
|
||||
'border-radius:10px',
|
||||
'background:rgba(17,24,39,.96)',
|
||||
'color:white',
|
||||
'font-family:system-ui,-apple-system,BlinkMacSystemFont,sans-serif',
|
||||
'box-shadow:0 12px 32px rgba(0,0,0,.35)',
|
||||
].join(';');
|
||||
panel.innerHTML = `
|
||||
<div style="display:flex;justify-content:space-between;gap:12px;align-items:center;margin-bottom:12px;">
|
||||
<strong style="font-size:15px;color:#93c5fd;">AI 原生转接</strong>
|
||||
<span id="static-copilot-status" style="font-size:12px;color:#d1d5db;">未连接</span>
|
||||
</div>
|
||||
<label style="display:block;font-size:12px;color:#9ca3af;margin-bottom:4px;">信令地址</label>
|
||||
<input id="static-copilot-url" value="${DEFAULT_SIGNALING_URL}" style="width:100%;box-sizing:border-box;margin-bottom:10px;padding:8px;border:1px solid #4b5563;border-radius:6px;background:#111827;color:white;">
|
||||
<label style="display:block;font-size:12px;color:#9ca3af;margin-bottom:4px;">助手 ID</label>
|
||||
<input id="static-copilot-assistant" placeholder="请输入助手 ID" style="width:100%;box-sizing:border-box;margin-bottom:12px;padding:8px;border:1px solid #4b5563;border-radius:6px;background:#111827;color:white;">
|
||||
<div style="margin-bottom:10px;">
|
||||
<div style="display:flex;justify-content:space-between;font-size:12px;color:#d1fae5;margin-bottom:4px;"><span>远端输入</span><span>原生音轨</span></div>
|
||||
<div style="height:8px;border-radius:999px;overflow:hidden;background:#374151;"><div id="static-copilot-input-vu" style="height:100%;width:0%;background:linear-gradient(90deg,#10b981,#fcd34d);transition:width .06s linear;"></div></div>
|
||||
</div>
|
||||
<div style="margin-bottom:12px;">
|
||||
<div style="display:flex;justify-content:space-between;font-size:12px;color:#bfdbfe;margin-bottom:4px;"><span>AI 输出</span><span>业务上行</span></div>
|
||||
<div style="height:8px;border-radius:999px;overflow:hidden;background:#374151;"><div id="static-copilot-output-vu" style="height:100%;width:0%;background:linear-gradient(90deg,#60a5fa,#c084fc);transition:width .06s linear;"></div></div>
|
||||
</div>
|
||||
<button id="static-copilot-toggle" style="width:100%;padding:12px;border:0;border-radius:6px;background:#10b981;color:white;font-weight:700;cursor:pointer;">当前：人工接管｜点击切换 AI</button>
|
||||
<button id="static-copilot-record" style="width:100%;margin-top:8px;padding:10px;border:0;border-radius:6px;background:#2563eb;color:white;font-weight:700;cursor:pointer;">录制 AI 输入</button>
|
||||
<div id="static-copilot-log" style="height:170px;margin-top:12px;padding:8px;overflow:auto;border:1px solid #374151;border-radius:6px;background:#030712;color:#a7f3d0;font:11px ui-monospace,Menlo,monospace;">
|
||||
<span data-placeholder="1" style="color:#6b7280;">等待交互...</span>
|
||||
</div>
|
||||
`;
|
||||
document.body.appendChild(panel);
|
||||
|
||||
const toggle = document.getElementById('static-copilot-toggle');
|
||||
const recordButton = document.getElementById('static-copilot-record');
|
||||
const urlInput = document.getElementById('static-copilot-url');
|
||||
const assistantInput = document.getElementById('static-copilot-assistant');
|
||||
recordButton.addEventListener('click', toggleInputRecording);
|
||||
toggle.addEventListener('click', async () => {
|
||||
if (!isAiTakeover && !assistantInput.value.trim()) {
|
||||
alert('请先填写助手 ID。');
|
||||
return;
|
||||
}
|
||||
isAiTakeover = !isAiTakeover;
|
||||
updateModeButton();
|
||||
if (isAiTakeover) {
|
||||
await connectAi(urlInput.value.trim(), assistantInput.value.trim());
|
||||
await attachAiOutputToBusiness();
|
||||
} else {
|
||||
await restoreBusinessAudio();
|
||||
disconnectAi();
|
||||
}
|
||||
});
|
||||
updateModeButton();
|
||||
updateRecordButton();
|
||||
}
|
||||
|
||||
function renderPanelSoon() {
|
||||
try { renderPanel(); } catch (error) { console.warn('[AI Native Static] panel error', error); }
|
||||
}
|
||||
|
||||
if (document.readyState === 'loading') {
|
||||
document.addEventListener('DOMContentLoaded', renderPanelSoon, { once: true });
|
||||
} else {
|
||||
renderPanelSoon();
|
||||
}
|
||||
requestAnimationFrame(renderVu);
|
||||
setInterval(refreshStats, 250);
|
||||
setTimeout(renderPanelSoon, 1500);
|
||||
setTimeout(renderPanelSoon, 3000);
|
||||
window.addEventListener('beforeunload', () => {
|
||||
if (inputRecorder?.state === 'recording') stopInputRecording();
|
||||
});
|
||||
})();
|
||||
Reference in New Issue
Block a user