1944 lines
85 KiB
JavaScript
1944 lines
85 KiB
JavaScript
// ==UserScript==
|
||
// @name 12345 AI Element2 Gated Copilot
|
||
// @namespace http://tampermonkey.net/
|
||
// @version 1.0
|
||
// @description Two-party gated bridge: only feed participant audio into AI after both sides are connected.
|
||
// @author You
|
||
// @match *://yhy-yw.22sj.cn:*/*
|
||
// @grant none
|
||
// @run-at document-start
|
||
// ==/UserScript==
|
||
|
||
(function () {
|
||
'use strict';
|
||
|
||
const DEFAULT_API_BASE_URL = 'http://182.92.86.220:8000';
|
||
const STATIC_AUDIO_ELEMENT_REASON = 'element-2:audio';
|
||
const STATIC_VIDEO_ELEMENT_REASON = 'element-0:video';
|
||
const PARTICIPANT_AUDIO_TRACK_PATTERN = /-audio-3-a$/;
|
||
|
||
const originalAddEventListener = RTCPeerConnection.prototype.addEventListener;
|
||
const originalAddTrack = RTCPeerConnection.prototype.addTrack;
|
||
const originalAddTransceiver = RTCPeerConnection.prototype.addTransceiver;
|
||
const originalSetRemoteDescription = RTCPeerConnection.prototype.setRemoteDescription;
|
||
const mediaSrcObjectDescriptor = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, 'srcObject');
|
||
|
||
const remoteAudio = new Map();
|
||
const remoteVideo = new Map();
|
||
const businessSenders = new Map();
|
||
const aiOutputTrackIds = new Set();
|
||
const inboundMemory = new Map();
|
||
const outboundMemory = new Map();
|
||
const internalPcs = new WeakSet();
|
||
const AUDIO_UPLINK_STORAGE_KEY = 'element2-copilot-send-audio';
|
||
const VIDEO_UPLINK_STORAGE_KEY = 'element2-copilot-send-video';
|
||
const AUDIO_SELECTION_MODE_STORAGE_KEY = 'element2-copilot-audio-selection-mode';
|
||
const VIDEO_SELECTION_MODE_STORAGE_KEY = 'element2-copilot-video-selection-mode';
|
||
const SENDER_SELECTION_MODE_STORAGE_KEY = 'element2-copilot-sender-selection-mode';
|
||
|
||
let selectedRemoteAudio = null;
|
||
let audioSelectionMode = readAudioSelectionMode();
|
||
let manualAudioTrackId = null;
|
||
let audioTrackSelectSignature = '';
|
||
let selectedRemoteVideo = null;
|
||
let videoSelectionMode = readVideoSelectionMode();
|
||
let manualVideoTrackId = null;
|
||
let videoTrackSelectSignature = '';
|
||
let selectedBusinessSender = null;
|
||
let senderSelectionMode = readSenderSelectionMode();
|
||
let manualSenderId = null;
|
||
let senderSelectSignature = '';
|
||
let changingBusinessSender = false;
|
||
let originalBusinessTrack = null;
|
||
let aiAudioTrack = null;
|
||
|
||
let aiPc = null;
|
||
let aiDc = null;
|
||
let aiAudioSender = null;
|
||
let aiVideoSender = null;
|
||
let aiPcId = null;
|
||
let aiOfferUrl = null;
|
||
let aiAssistantId = null;
|
||
let aiAuthorization = null;
|
||
let aiCandidateQueue = [];
|
||
let aiCandidateFlushTimer = null;
|
||
let aiKeepAliveTimer = null;
|
||
let aiNegotiating = false;
|
||
let aiCanSendCandidates = false;
|
||
let isAiTakeover = false;
|
||
let aiAttached = false;
|
||
let aiOutputAudioContext = null;
|
||
let aiOutputAnalyser = null;
|
||
let aiOutputAnalyserData = null;
|
||
let aiOutputAnalyserSource = null;
|
||
let aiOutputAnalyserTrack = null;
|
||
let inputVu = 0;
|
||
let outputVu = 0;
|
||
let inputHealthTimer = null;
|
||
let outputHealthTimer = null;
|
||
let assistantLine = null;
|
||
let assistantBuffer = '';
|
||
let inputRecorder = null;
|
||
let inputRecordingChunks = [];
|
||
let inputRecordingTrack = null;
|
||
let inputRecordingStartedAt = 0;
|
||
let senderRecorder = null;
|
||
let senderRecordingChunks = [];
|
||
let senderRecordingTrack = null;
|
||
let senderRecordingStartedAt = 0;
|
||
let videoRecorder = null;
|
||
let videoRecordingChunks = [];
|
||
let videoRecordingTrack = null;
|
||
let videoRecordingStartedAt = 0;
|
||
|
||
function log(message) {
|
||
console.log(`[AI Element2 Gated] ${message}`);
|
||
}
|
||
|
||
function readBoolSetting(key, fallback) {
|
||
try {
|
||
const value = localStorage.getItem(key);
|
||
if (value === null) return fallback;
|
||
return value === 'true';
|
||
} catch {
|
||
return fallback;
|
||
}
|
||
}
|
||
|
||
function writeBoolSetting(key, value) {
|
||
try {
|
||
localStorage.setItem(key, value ? 'true' : 'false');
|
||
} catch {
|
||
// Ignore storage failures; the current checkbox state still applies.
|
||
}
|
||
}
|
||
|
||
function readAudioSelectionMode() {
|
||
try {
|
||
if (!shouldSendAudioToAi()) return 'off';
|
||
return localStorage.getItem(AUDIO_SELECTION_MODE_STORAGE_KEY) === 'manual' ? 'manual' : 'auto';
|
||
} catch {
|
||
return 'auto';
|
||
}
|
||
}
|
||
|
||
function writeAudioSelectionMode(mode) {
|
||
audioSelectionMode = ['auto', 'manual', 'off'].includes(mode) ? mode : 'auto';
|
||
writeBoolSetting(AUDIO_UPLINK_STORAGE_KEY, audioSelectionMode !== 'off');
|
||
try {
|
||
localStorage.setItem(AUDIO_SELECTION_MODE_STORAGE_KEY, audioSelectionMode);
|
||
} catch {}
|
||
}
|
||
|
||
function readVideoSelectionMode() {
|
||
try {
|
||
if (!shouldSendVideoToAi()) return 'off';
|
||
return localStorage.getItem(VIDEO_SELECTION_MODE_STORAGE_KEY) === 'manual' ? 'manual' : 'auto';
|
||
} catch {
|
||
return 'auto';
|
||
}
|
||
}
|
||
|
||
function writeVideoSelectionMode(mode) {
|
||
videoSelectionMode = ['auto', 'manual', 'off'].includes(mode) ? mode : 'auto';
|
||
writeBoolSetting(VIDEO_UPLINK_STORAGE_KEY, videoSelectionMode !== 'off');
|
||
try {
|
||
localStorage.setItem(VIDEO_SELECTION_MODE_STORAGE_KEY, videoSelectionMode);
|
||
} catch {}
|
||
}
|
||
|
||
function readSenderSelectionMode() {
|
||
try {
|
||
return localStorage.getItem(SENDER_SELECTION_MODE_STORAGE_KEY) === 'manual' ? 'manual' : 'auto';
|
||
} catch {
|
||
return 'auto';
|
||
}
|
||
}
|
||
|
||
function writeSenderSelectionMode(mode) {
|
||
senderSelectionMode = mode === 'manual' ? 'manual' : 'auto';
|
||
try {
|
||
localStorage.setItem(SENDER_SELECTION_MODE_STORAGE_KEY, senderSelectionMode);
|
||
} catch {}
|
||
}
|
||
|
||
function shouldSendAudioToAi() {
|
||
return readBoolSetting(AUDIO_UPLINK_STORAGE_KEY, true);
|
||
}
|
||
|
||
function shouldSendVideoToAi() {
|
||
return readBoolSetting(VIDEO_UPLINK_STORAGE_KEY, true);
|
||
}
|
||
|
||
function appendChatLine(role, text, important = false) {
|
||
const box = document.getElementById('element2-copilot-log');
|
||
if (!box) return;
|
||
if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = '';
|
||
const row = document.createElement('div');
|
||
row.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] ${role}: ${text}`;
|
||
row.style.marginBottom = '4px';
|
||
if (important) row.style.color = '#fcd34d';
|
||
box.appendChild(row);
|
||
while (box.children.length > 120) box.removeChild(box.firstChild);
|
||
box.scrollTop = box.scrollHeight;
|
||
return row;
|
||
}
|
||
|
||
function startAssistantLine() {
|
||
assistantBuffer = '';
|
||
assistantLine = null;
|
||
}
|
||
|
||
function appendAssistantDelta(delta) {
|
||
if (!delta) return;
|
||
const box = document.getElementById('element2-copilot-log');
|
||
if (!box) {
|
||
console.log(`[AI Element2 Gated] AI: ${delta}`);
|
||
return;
|
||
}
|
||
if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = '';
|
||
if (!assistantLine) assistantLine = appendChatLine('AI', '');
|
||
assistantBuffer += delta;
|
||
assistantLine.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] AI: ${assistantBuffer}`;
|
||
box.scrollTop = box.scrollHeight;
|
||
}
|
||
|
||
function finishAssistantLine() {
|
||
assistantLine = null;
|
||
assistantBuffer = '';
|
||
}
|
||
|
||
function logUserTranscript(text) {
|
||
if (!text?.trim()) return;
|
||
finishAssistantLine();
|
||
appendChatLine('用户', text.trim(), true);
|
||
}
|
||
|
||
function recorderMimeType() {
|
||
if (window.MediaRecorder?.isTypeSupported?.('audio/webm;codecs=opus')) return 'audio/webm;codecs=opus';
|
||
if (window.MediaRecorder?.isTypeSupported?.('audio/webm')) return 'audio/webm';
|
||
return '';
|
||
}
|
||
|
||
function recordingFileName(track) {
|
||
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||
const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track';
|
||
return `ai-input-${trackPart}-${stamp}.webm`;
|
||
}
|
||
|
||
function senderRecordingFileName(track) {
|
||
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||
const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track';
|
||
return `business-sender-${trackPart}-${stamp}.webm`;
|
||
}
|
||
|
||
function videoRecordingFileName(track) {
|
||
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||
const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track';
|
||
return `ai-video-${trackPart}-${stamp}.webm`;
|
||
}
|
||
|
||
function videoRecorderMimeType() {
|
||
if (window.MediaRecorder?.isTypeSupported?.('video/webm;codecs=vp8')) return 'video/webm;codecs=vp8';
|
||
if (window.MediaRecorder?.isTypeSupported?.('video/webm')) return 'video/webm';
|
||
return '';
|
||
}
|
||
|
||
async function startInputRecording() {
|
||
if (inputRecorder?.state === 'recording') return;
|
||
selectRemoteAudio('record');
|
||
if (!isLive(selectedRemoteAudio)) {
|
||
appendChatLine('录音', `没有可录制的 ${STATIC_AUDIO_ELEMENT_REASON} 音轨`, true);
|
||
return;
|
||
}
|
||
if (!window.MediaRecorder) {
|
||
appendChatLine('录音', '当前浏览器不支持 MediaRecorder', true);
|
||
return;
|
||
}
|
||
|
||
inputRecordingChunks = [];
|
||
inputRecordingTrack = selectedRemoteAudio;
|
||
inputRecordingStartedAt = Date.now();
|
||
const stream = new MediaStream([selectedRemoteAudio]);
|
||
const mimeType = recorderMimeType();
|
||
inputRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined);
|
||
inputRecorder.ondataavailable = (event) => {
|
||
if (event.data?.size > 0) inputRecordingChunks.push(event.data);
|
||
};
|
||
inputRecorder.onstop = () => {
|
||
const blob = new Blob(inputRecordingChunks, { type: inputRecorder?.mimeType || 'audio/webm' });
|
||
const url = URL.createObjectURL(blob);
|
||
const link = document.createElement('a');
|
||
link.href = url;
|
||
link.download = recordingFileName(inputRecordingTrack);
|
||
document.body.appendChild(link);
|
||
link.click();
|
||
link.remove();
|
||
setTimeout(() => URL.revokeObjectURL(url), 30000);
|
||
|
||
const seconds = ((Date.now() - inputRecordingStartedAt) / 1000).toFixed(1);
|
||
appendChatLine('录音', `已保存 ${link.download}（${seconds}s，${(blob.size / 1024).toFixed(1)} KB）`, true);
|
||
inputRecorder = null;
|
||
inputRecordingChunks = [];
|
||
inputRecordingTrack = null;
|
||
updateRecordButton();
|
||
};
|
||
inputRecorder.start(250);
|
||
appendChatLine('录音', `开始录制 AI 输入：${trackLabel(inputRecordingTrack)}`, true);
|
||
updateRecordButton();
|
||
}
|
||
|
||
function stopInputRecording() {
|
||
if (inputRecorder?.state === 'recording') inputRecorder.stop();
|
||
}
|
||
|
||
function toggleInputRecording() {
|
||
if (inputRecorder?.state === 'recording') stopInputRecording();
|
||
else startInputRecording();
|
||
}
|
||
|
||
function updateRecordButton() {
|
||
const button = document.getElementById('element2-copilot-record');
|
||
if (!button) return;
|
||
const recording = inputRecorder?.state === 'recording';
|
||
button.textContent = recording ? '■' : '●';
|
||
button.title = recording ? '停止录制 AI 输入' : '录制 AI 输入';
|
||
button.setAttribute('aria-label', button.title);
|
||
button.style.background = recording ? '#dc2626' : '#2563eb';
|
||
}
|
||
|
||
async function startSenderRecording() {
|
||
if (senderRecorder?.state === 'recording') return;
|
||
selectBusinessSender('record-sender');
|
||
const track = selectedBusinessSender?.track;
|
||
if (!isLive(track)) {
|
||
appendChatLine('录音', '当前没有可录制的业务 sender 音轨', true);
|
||
return;
|
||
}
|
||
if (!window.MediaRecorder) {
|
||
appendChatLine('录音', '当前浏览器不支持 MediaRecorder', true);
|
||
return;
|
||
}
|
||
senderRecordingChunks = [];
|
||
senderRecordingTrack = track;
|
||
senderRecordingStartedAt = Date.now();
|
||
const mimeType = recorderMimeType();
|
||
senderRecorder = new MediaRecorder(new MediaStream([track]), mimeType ? { mimeType } : undefined);
|
||
senderRecorder.ondataavailable = (event) => {
|
||
if (event.data?.size > 0) senderRecordingChunks.push(event.data);
|
||
};
|
||
senderRecorder.onstop = () => {
|
||
const blob = new Blob(senderRecordingChunks, { type: senderRecorder?.mimeType || 'audio/webm' });
|
||
const url = URL.createObjectURL(blob);
|
||
const link = document.createElement('a');
|
||
link.href = url;
|
||
link.download = senderRecordingFileName(senderRecordingTrack);
|
||
document.body.appendChild(link);
|
||
link.click();
|
||
link.remove();
|
||
setTimeout(() => URL.revokeObjectURL(url), 30000);
|
||
const seconds = ((Date.now() - senderRecordingStartedAt) / 1000).toFixed(1);
|
||
appendChatLine('录音', `已保存 ${link.download}（${seconds}s，${(blob.size / 1024).toFixed(1)} KB）`, true);
|
||
senderRecorder = null;
|
||
senderRecordingChunks = [];
|
||
senderRecordingTrack = null;
|
||
updateSenderRecordButton();
|
||
};
|
||
senderRecorder.start(250);
|
||
appendChatLine('录音', `开始录制业务 sender：${trackLabel(track)}`, true);
|
||
updateSenderRecordButton();
|
||
}
|
||
|
||
function toggleSenderRecording() {
|
||
if (senderRecorder?.state === 'recording') senderRecorder.stop();
|
||
else void startSenderRecording();
|
||
}
|
||
|
||
function updateSenderRecordButton() {
|
||
const button = document.getElementById('element2-copilot-record-sender');
|
||
if (!button) return;
|
||
const recording = senderRecorder?.state === 'recording';
|
||
button.textContent = recording ? '■' : '●';
|
||
button.title = recording ? '停止录制 sender 音轨' : '录制 sender 音轨';
|
||
button.setAttribute('aria-label', button.title);
|
||
button.style.background = recording ? '#dc2626' : '#7c3aed';
|
||
}
|
||
|
||
function updateVideoRecordButton() {
|
||
const button = document.getElementById('element2-copilot-record-video');
|
||
if (!button) return;
|
||
const recording = videoRecorder?.state === 'recording';
|
||
button.textContent = recording ? '■' : '●';
|
||
button.title = recording ? '停止录制 AI 视频' : '录制 AI 视频';
|
||
button.setAttribute('aria-label', button.title);
|
||
button.style.background = recording ? '#dc2626' : '#475569';
|
||
}
|
||
|
||
async function startVideoRecording() {
|
||
if (videoRecorder?.state === 'recording') return;
|
||
selectRemoteVideo('record-video');
|
||
if (!isLive(selectedRemoteVideo)) {
|
||
appendChatLine('录制视频', `没有可录制的 ${STATIC_VIDEO_ELEMENT_REASON} 视频轨`, true);
|
||
return;
|
||
}
|
||
if (!window.MediaRecorder) {
|
||
appendChatLine('录制视频', '当前浏览器不支持 MediaRecorder', true);
|
||
return;
|
||
}
|
||
|
||
videoRecordingChunks = [];
|
||
videoRecordingTrack = selectedRemoteVideo;
|
||
videoRecordingStartedAt = Date.now();
|
||
const stream = new MediaStream([videoRecordingTrack]);
|
||
const mimeType = videoRecorderMimeType();
|
||
videoRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined);
|
||
videoRecorder.ondataavailable = (event) => {
|
||
if (event.data?.size > 0) videoRecordingChunks.push(event.data);
|
||
};
|
||
videoRecorder.onstop = () => {
|
||
const blob = new Blob(videoRecordingChunks, { type: videoRecorder?.mimeType || 'video/webm' });
|
||
const url = URL.createObjectURL(blob);
|
||
const link = document.createElement('a');
|
||
link.href = url;
|
||
link.download = videoRecordingFileName(videoRecordingTrack);
|
||
document.body.appendChild(link);
|
||
link.click();
|
||
link.remove();
|
||
setTimeout(() => URL.revokeObjectURL(url), 30000);
|
||
|
||
const seconds = ((Date.now() - videoRecordingStartedAt) / 1000).toFixed(1);
|
||
appendChatLine('录制视频', `已保存 ${link.download}（${seconds}s，${(blob.size / 1024).toFixed(1)} KB）`, true);
|
||
videoRecorder = null;
|
||
videoRecordingChunks = [];
|
||
videoRecordingTrack = null;
|
||
updateVideoRecordButton();
|
||
};
|
||
videoRecorder.start(250);
|
||
appendChatLine('录制视频', `开始录制 AI 视频：${trackLabel(videoRecordingTrack)}`, true);
|
||
updateVideoRecordButton();
|
||
}
|
||
|
||
function stopVideoRecording() {
|
||
if (videoRecorder?.state === 'recording') videoRecorder.stop();
|
||
}
|
||
|
||
function toggleVideoRecording() {
|
||
if (videoRecorder?.state === 'recording') stopVideoRecording();
|
||
else startVideoRecording();
|
||
}
|
||
|
||
function pcId() {
|
||
const bytes = new Uint8Array(16);
|
||
crypto.getRandomValues(bytes);
|
||
return 'PC-' + Array.from(bytes).map((b) => b.toString(16).padStart(2, '0')).join('');
|
||
}
|
||
|
||
function isInternalPc(pc) {
|
||
return Boolean(pc && (pc.__aiElement2GatedInternal || internalPcs.has(pc) || pc === aiPc));
|
||
}
|
||
|
||
function isLive(track) {
|
||
return track && track.readyState === 'live';
|
||
}
|
||
|
||
function trackLabel(track) {
|
||
if (!track) return 'none';
|
||
return `${track.id}${track.muted ? ' muted' : ''}`;
|
||
}
|
||
|
||
function markInternalPc(pc) {
|
||
if (!pc) return;
|
||
pc.__aiElement2GatedInternal = true;
|
||
internalPcs.add(pc);
|
||
}
|
||
|
||
function elementReasonFor(element) {
|
||
if (!element) return null;
|
||
const tag = element.tagName?.toLowerCase();
|
||
if (tag !== 'audio' && tag !== 'video') return null;
|
||
const elements = Array.from(document.querySelectorAll(tag));
|
||
const index = elements.indexOf(element);
|
||
if (index < 0) return `${tag}-srcObject`;
|
||
return `element-${index}:${tag}`;
|
||
}
|
||
|
||
function hasElementReason(item, reason) {
|
||
return Boolean(item?.reasons?.has(reason));
|
||
}
|
||
|
||
function elementAudioReasons(item) {
|
||
return Array.from(item?.reasons || []).filter((reason) => /^element-\d+:audio$/.test(reason));
|
||
}
|
||
|
||
function audioSourceLabel(item) {
|
||
const reasons = elementAudioReasons(item);
|
||
return reasons[0] || 'audio-track';
|
||
}
|
||
|
||
function shortTrackId(track) {
|
||
if (!track?.id) return 'none';
|
||
return track.id.length <= 22 ? track.id : `${track.id.slice(0, 10)}...${track.id.slice(-7)}`;
|
||
}
|
||
|
||
function selectableAudioItems() {
|
||
return Array.from(remoteAudio.values()).filter((item) => (
|
||
item?.receiver
|
||
&& isLive(item.track)
|
||
&& !aiOutputTrackIds.has(item.track.id)
|
||
));
|
||
}
|
||
|
||
function audioOptionLabel(item) {
|
||
const source = audioSourceLabel(item);
|
||
const state = item.track.muted ? '静音' : '可用';
|
||
return `${source} · ${shortTrackId(item.track)} · ${state}`;
|
||
}
|
||
|
||
function updateAudioTrackSelect() {
|
||
const select = document.getElementById('element2-copilot-audio-track');
|
||
if (!select) return;
|
||
const items = selectableAudioItems().sort((a, b) => scoreAudioCandidate(b) - scoreAudioCandidate(a));
|
||
const desiredValue = audioSelectionMode === 'manual' ? manualAudioTrackId : selectedRemoteAudio?.id;
|
||
const signature = [
|
||
audioSelectionMode,
|
||
desiredValue || '',
|
||
...items.map((item) => `${item.track.id}:${item.track.muted}:${audioSourceLabel(item)}`),
|
||
].join('|');
|
||
if (signature === audioTrackSelectSignature) return;
|
||
audioTrackSelectSignature = signature;
|
||
select.innerHTML = '';
|
||
if (items.length === 0) {
|
||
const option = document.createElement('option');
|
||
option.value = '';
|
||
option.textContent = '暂无可用远端音轨';
|
||
select.appendChild(option);
|
||
} else {
|
||
if (audioSelectionMode === 'manual' && !manualAudioTrackId) {
|
||
const placeholder = document.createElement('option');
|
||
placeholder.value = '';
|
||
placeholder.textContent = '请选择远端音轨';
|
||
select.appendChild(placeholder);
|
||
}
|
||
for (const item of items) {
|
||
const option = document.createElement('option');
|
||
option.value = item.track.id;
|
||
option.textContent = audioOptionLabel(item);
|
||
select.appendChild(option);
|
||
}
|
||
}
|
||
if (desiredValue && items.some((item) => item.track.id === desiredValue)) select.value = desiredValue;
|
||
select.disabled = audioSelectionMode !== 'manual';
|
||
select.style.opacity = select.disabled ? '0.55' : '1';
|
||
}
|
||
|
||
function updateSelectedAudioLabel() {
|
||
const label = document.getElementById('element2-copilot-selected-audio');
|
||
if (!label) return;
|
||
const mode = audioSelectionMode === 'manual' ? '手动' : audioSelectionMode === 'off' ? '关闭' : '自动';
|
||
const item = selectedRemoteAudio ? remoteAudio.get(selectedRemoteAudio.id) : null;
|
||
label.textContent = selectedRemoteAudio
|
||
? `${mode}：${audioSourceLabel(item)} · ${shortTrackId(selectedRemoteAudio)}`
|
||
: audioSelectionMode === 'off' ? '不传音频给 AI' : `${mode}：尚未选中音轨`;
|
||
}
|
||
|
||
function pageHasParticipant() {
|
||
const text = document.body?.innerText || '';
|
||
return text.includes('当事人');
|
||
}
|
||
|
||
function pageHasOperator() {
|
||
const text = document.body?.innerText || '';
|
||
return text.includes('坐席端');
|
||
}
|
||
|
||
function selectedAudioItem() {
|
||
const item = selectedRemoteAudio ? remoteAudio.get(selectedRemoteAudio.id) : null;
|
||
return item?.receiver && isLive(item.track) && !item.track.muted ? item : null;
|
||
}
|
||
|
||
function gateStatus() {
|
||
const item = selectedAudioItem();
|
||
const participantAudioPresent = selectableAudioItems().some((candidate) => !candidate.track.muted);
|
||
const senderItem = Array.from(businessSenders.values()).find((candidate) => (
|
||
candidate.sender === selectedBusinessSender
|
||
&& selectableSenderItems().includes(candidate)
|
||
));
|
||
const participantVisible = pageHasParticipant();
|
||
const operatorVisible = pageHasOperator();
|
||
return {
|
||
bothParties: participantVisible && operatorVisible,
|
||
participantOnline: participantVisible && (Boolean(item) || (audioSelectionMode === 'off' && participantAudioPresent)),
|
||
operatorOnline: operatorVisible,
|
||
senderReady: Boolean(senderItem),
|
||
track: item?.track || null,
|
||
};
|
||
}
|
||
|
||
function gateMessage(status = gateStatus()) {
|
||
if (!status.operatorOnline) return '等待坐席在线';
|
||
if (!status.participantOnline) return '等待当事人在线';
|
||
if (!status.senderReady) return '等待通话上行就绪';
|
||
return '就绪：可开启 AI 托管';
|
||
}
|
||
|
||
function updateGateLabel() {
|
||
const label = document.getElementById('element2-copilot-gate');
|
||
if (!label) return;
|
||
const status = gateStatus();
|
||
label.textContent = gateMessage(status);
|
||
label.style.color = status.participantOnline && status.operatorOnline && status.senderReady ? '#86efac' : '#fcd34d';
|
||
}
|
||
|
||
function scoreAudioCandidate(item) {
|
||
if (!item || !isLive(item.track) || item.track.muted || aiOutputTrackIds.has(item.track.id)) return -1;
|
||
let score = 0;
|
||
if (item.receiver) score += 300;
|
||
if ((item.bytesReceived || 0) > 0) score += 100;
|
||
if ((item.bytesDelta || 0) > 0) score += 4000 + Math.min(1500, item.bytesDelta);
|
||
if ((item.packetsDelta || 0) > 0) score += 2000 + Math.min(800, item.packetsDelta * 8);
|
||
score += Math.min(1200, (item.audioLevel || 0) * 12000);
|
||
if (elementAudioReasons(item).length > 0) score += 100;
|
||
if (hasElementReason(item, STATIC_AUDIO_ELEMENT_REASON)) score += 5000;
|
||
if (PARTICIPANT_AUDIO_TRACK_PATTERN.test(item.track.id)) score += 5000;
|
||
item.audioScore = score;
|
||
return score;
|
||
}
|
||
|
||
function rememberRemoteReceiver(receiver, reason) {
|
||
const track = receiver?.track;
|
||
if (!isLive(track) || isInternalPc(receiver?.transport)) return;
|
||
|
||
if (track.kind === 'audio' && !aiOutputTrackIds.has(track.id)) {
|
||
let item = remoteAudio.get(track.id);
|
||
if (!item) {
|
||
item = { track, receiver, reasons: new Set(), bytesDelta: 0, packetsDelta: 0, audioLevel: 0, audioScore: 0 };
|
||
remoteAudio.set(track.id, item);
|
||
log(`audio candidate ${trackLabel(track)} (${reason})`, true);
|
||
}
|
||
item.track = track;
|
||
item.receiver = receiver || item.receiver;
|
||
item.reasons.add(reason);
|
||
selectRemoteAudio(reason);
|
||
}
|
||
|
||
if (track.kind === 'video') {
|
||
let item = remoteVideo.get(track.id);
|
||
if (!item) {
|
||
item = { track, receiver, reasons: new Set() };
|
||
remoteVideo.set(track.id, item);
|
||
log(`video candidate ${trackLabel(track)} (${reason})`, true);
|
||
}
|
||
item.track = track;
|
||
item.receiver = receiver || item.receiver;
|
||
item.reasons.add(reason);
|
||
selectRemoteVideo(reason);
|
||
}
|
||
}
|
||
|
||
function rememberRemoteStream(stream, reason) {
|
||
if (!stream || !(stream instanceof MediaStream)) return;
|
||
for (const track of stream.getAudioTracks()) {
|
||
if (aiOutputTrackIds.has(track.id)) continue;
|
||
let item = remoteAudio.get(track.id);
|
||
if (!item) {
|
||
item = { track, receiver: null, reasons: new Set(), bytesDelta: 0, packetsDelta: 0, audioLevel: 0, audioScore: 0 };
|
||
remoteAudio.set(track.id, item);
|
||
log(`audio stream candidate ${trackLabel(track)} (${reason})`, true);
|
||
}
|
||
item.track = track;
|
||
item.reasons.add(reason);
|
||
}
|
||
for (const track of stream.getVideoTracks()) {
|
||
let item = remoteVideo.get(track.id);
|
||
if (!item) {
|
||
item = { track, receiver: null, reasons: new Set() };
|
||
remoteVideo.set(track.id, item);
|
||
log(`video stream candidate ${trackLabel(track)} (${reason})`, true);
|
||
}
|
||
item.track = track;
|
||
item.reasons.add(reason);
|
||
}
|
||
selectRemoteAudio(reason);
|
||
selectRemoteVideo(reason);
|
||
}
|
||
|
||
function selectRemoteAudio(reason) {
|
||
const items = selectableAudioItems();
|
||
const current = selectedRemoteAudio && remoteAudio.get(selectedRemoteAudio.id);
|
||
let item = null;
|
||
if (audioSelectionMode === 'off') {
|
||
item = null;
|
||
} else if (audioSelectionMode === 'manual') {
|
||
item = items.find((candidate) => candidate.track.id === manualAudioTrackId) || null;
|
||
} else {
|
||
const ranked = items
|
||
.filter((candidate) => !candidate.track.muted)
|
||
.sort((a, b) => scoreAudioCandidate(b) - scoreAudioCandidate(a));
|
||
const best = ranked[0] || null;
|
||
const currentScore = current ? scoreAudioCandidate(current) : -1;
|
||
const bestScore = best ? scoreAudioCandidate(best) : -1;
|
||
item = current && currentScore >= 0 && (!best || bestScore < currentScore + 1500)
|
||
? current
|
||
: best;
|
||
}
|
||
if (!item) {
|
||
if (selectedRemoteAudio) {
|
||
selectedRemoteAudio = null;
|
||
if (aiAudioSender) void aiAudioSender.replaceTrack(null).catch(() => {});
|
||
}
|
||
updateSelectedAudioLabel();
|
||
updateAudioTrackSelect();
|
||
updateGateLabel();
|
||
return null;
|
||
}
|
||
|
||
if (selectedRemoteAudio !== item.track) {
|
||
selectedRemoteAudio = item.track;
|
||
log(`selected input audio (${reason}) from ${audioSourceLabel(item)}: ${trackLabel(selectedRemoteAudio)}`, true);
|
||
updateSelectedAudioLabel();
|
||
attachAudioToAi();
|
||
}
|
||
updateAudioTrackSelect();
|
||
updateGateLabel();
|
||
return selectedRemoteAudio;
|
||
}
|
||
|
||
function selectRemoteVideo(reason) {
|
||
const items = selectableVideoItems();
|
||
let item = null;
|
||
if (videoSelectionMode === 'manual') {
|
||
item = items.find((candidate) => candidate.track.id === manualVideoTrackId) || null;
|
||
} else if (videoSelectionMode === 'auto') {
|
||
item = items.find((candidate) => hasElementReason(candidate, STATIC_VIDEO_ELEMENT_REASON)) || items[0] || null;
|
||
}
|
||
if (!item) {
|
||
if (selectedRemoteVideo) {
|
||
selectedRemoteVideo = null;
|
||
if (aiVideoSender) void aiVideoSender.replaceTrack(null).catch(() => {});
|
||
}
|
||
updateVideoTrackSelect();
|
||
return null;
|
||
}
|
||
|
||
if (selectedRemoteVideo !== item.track) {
|
||
selectedRemoteVideo = item.track;
|
||
log(`selected input video (${reason}): ${trackLabel(selectedRemoteVideo)}`, true);
|
||
attachVideoToAi();
|
||
}
|
||
updateVideoTrackSelect();
|
||
return selectedRemoteVideo;
|
||
}
|
||
|
||
function selectableVideoItems() {
|
||
return Array.from(remoteVideo.values()).filter((item) => (
|
||
item?.receiver
|
||
&& isLive(item.track)
|
||
&& !item.track.muted
|
||
));
|
||
}
|
||
|
||
function videoSourceLabel(item) {
|
||
const reason = Array.from(item?.reasons || []).find((value) => /^element-\d+:video$/.test(value));
|
||
return reason || 'video-track';
|
||
}
|
||
|
||
function updateVideoTrackSelect() {
|
||
const select = document.getElementById('element2-copilot-video-track');
|
||
if (!select) return;
|
||
const items = selectableVideoItems();
|
||
const desiredValue = videoSelectionMode === 'manual' ? manualVideoTrackId : selectedRemoteVideo?.id;
|
||
const signature = [
|
||
videoSelectionMode,
|
||
desiredValue || '',
|
||
...items.map((item) => `${item.track.id}:${videoSourceLabel(item)}`),
|
||
].join('|');
|
||
if (signature === videoTrackSelectSignature) return;
|
||
videoTrackSelectSignature = signature;
|
||
select.innerHTML = '';
|
||
if (items.length === 0) {
|
||
const option = document.createElement('option');
|
||
option.value = '';
|
||
option.textContent = '暂无可用远端视频';
|
||
select.appendChild(option);
|
||
} else {
|
||
if (videoSelectionMode === 'manual' && !manualVideoTrackId) {
|
||
const placeholder = document.createElement('option');
|
||
placeholder.value = '';
|
||
placeholder.textContent = '请选择远端视频';
|
||
select.appendChild(placeholder);
|
||
}
|
||
for (const item of items) {
|
||
const option = document.createElement('option');
|
||
option.value = item.track.id;
|
||
option.textContent = `${videoSourceLabel(item)} · ${shortTrackId(item.track)}`;
|
||
select.appendChild(option);
|
||
}
|
||
}
|
||
if (desiredValue && items.some((item) => item.track.id === desiredValue)) select.value = desiredValue;
|
||
select.disabled = videoSelectionMode !== 'manual';
|
||
select.style.opacity = select.disabled ? '0.55' : '1';
|
||
}
|
||
|
||
function rememberBusinessSender(sender, track, reason) {
|
||
if (!sender) return;
|
||
const senderTrack = track || sender.track;
|
||
if (senderTrack && senderTrack.kind !== 'audio') return;
|
||
|
||
const id = sender.__aiElement2GatedSenderId || `sender-${businessSenders.size + 1}`;
|
||
sender.__aiElement2GatedSenderId = id;
|
||
|
||
let item = businessSenders.get(id);
|
||
if (!item) {
|
||
item = { id, sender, track: senderTrack || null, originalTrack: senderTrack || null, sources: new Set(), bytesDelta: 0, packetsDelta: 0 };
|
||
businessSenders.set(id, item);
|
||
log(`business sender candidate ${id} (${reason}): ${trackLabel(senderTrack)}`, true);
|
||
}
|
||
item.sender = sender;
|
||
item.track = sender.track || senderTrack || item.track;
|
||
if (!isAiTakeover && senderTrack && senderTrack !== aiAudioTrack) item.originalTrack = senderTrack;
|
||
item.sources.add(reason);
|
||
wrapBusinessSender(sender);
|
||
selectBusinessSender(reason);
|
||
}
|
||
|
||
function wrapBusinessSender(sender) {
|
||
if (!sender || sender.__aiElement2GatedWrapped) return;
|
||
const originalReplaceTrack = sender.replaceTrack?.bind(sender);
|
||
if (!originalReplaceTrack) return;
|
||
|
||
sender.replaceTrack = async function (track) {
|
||
const result = await originalReplaceTrack(track);
|
||
rememberBusinessSender(sender, track, 'replaceTrack');
|
||
return result;
|
||
};
|
||
sender.__aiElement2GatedWrapped = true;
|
||
}
|
||
|
||
function isRemoteInputTrack(track) {
|
||
const item = track ? remoteAudio.get(track.id) : null;
|
||
return Boolean(item?.receiver && !aiOutputTrackIds.has(track.id));
|
||
}
|
||
|
||
function senderCandidateTrack(item) {
|
||
const current = item?.sender?.track;
|
||
if (current && current !== aiAudioTrack) return current;
|
||
return item?.originalTrack || item?.track || null;
|
||
}
|
||
|
||
function selectableSenderItems() {
|
||
return Array.from(businessSenders.values()).filter((item) => {
|
||
const track = senderCandidateTrack(item);
|
||
return isLive(track) && track.kind === 'audio' && !isRemoteInputTrack(track);
|
||
});
|
||
}
|
||
|
||
function scoreSenderCandidate(item) {
|
||
const track = senderCandidateTrack(item);
|
||
if (!isLive(track) || isRemoteInputTrack(track)) return -1;
|
||
let score = 1000;
|
||
if (!track.muted) score += 400;
|
||
if (item.sources.has('pc-addTrack')) score += 2000;
|
||
if ((item.bytesSent || 0) > 0) score += 300;
|
||
if ((item.bytesDelta || 0) > 0) score += 4500 + Math.min(1800, item.bytesDelta);
|
||
if ((item.packetsDelta || 0) > 0) score += 1800 + Math.min(900, item.packetsDelta * 10);
|
||
return score;
|
||
}
|
||
|
||
function updateSenderSelect() {
|
||
const select = document.getElementById('element2-copilot-sender');
|
||
if (!select) return;
|
||
const items = selectableSenderItems().sort((a, b) => scoreSenderCandidate(b) - scoreSenderCandidate(a));
|
||
const selectedItem = items.find((item) => item.sender === selectedBusinessSender);
|
||
const desiredValue = senderSelectionMode === 'manual' ? manualSenderId : selectedItem?.id;
|
||
const signature = [
|
||
senderSelectionMode,
|
||
desiredValue || '',
|
||
...items.map((item) => `${item.id}:${senderCandidateTrack(item)?.id || ''}:${senderCandidateTrack(item)?.muted}`),
|
||
].join('|');
|
||
if (signature === senderSelectSignature) return;
|
||
senderSelectSignature = signature;
|
||
select.innerHTML = '';
|
||
if (items.length === 0) {
|
||
const option = document.createElement('option');
|
||
option.value = '';
|
||
option.textContent = '暂无可用 sender';
|
||
select.appendChild(option);
|
||
} else {
|
||
if (senderSelectionMode === 'manual' && !manualSenderId) {
|
||
const placeholder = document.createElement('option');
|
||
placeholder.value = '';
|
||
placeholder.textContent = '请选择 sender';
|
||
select.appendChild(placeholder);
|
||
}
|
||
for (const item of items) {
|
||
const option = document.createElement('option');
|
||
option.value = item.id;
|
||
option.textContent = `${item.id} · ${shortTrackId(senderCandidateTrack(item))}${item.sources.has('pc-addTrack') ? ' · addTrack' : ''}`;
|
||
select.appendChild(option);
|
||
}
|
||
}
|
||
if (desiredValue && items.some((item) => item.id === desiredValue)) select.value = desiredValue;
|
||
select.disabled = senderSelectionMode !== 'manual';
|
||
select.style.opacity = select.disabled ? '0.55' : '1';
|
||
}
|
||
|
||
function selectBusinessSender(reason) {
|
||
if (changingBusinessSender) return selectedBusinessSender;
|
||
const items = selectableSenderItems();
|
||
const current = items.find((candidate) => candidate.sender === selectedBusinessSender);
|
||
let item = null;
|
||
if (senderSelectionMode === 'manual') {
|
||
item = items.find((candidate) => candidate.id === manualSenderId) || null;
|
||
} else {
|
||
item = current || items.sort((a, b) => scoreSenderCandidate(b) - scoreSenderCandidate(a))[0] || null;
|
||
}
|
||
if (!item) {
|
||
if (!aiAttached) {
|
||
selectedBusinessSender = null;
|
||
originalBusinessTrack = null;
|
||
}
|
||
updateSenderSelect();
|
||
return null;
|
||
}
|
||
if (selectedBusinessSender === item.sender) {
|
||
updateSenderSelect();
|
||
return selectedBusinessSender;
|
||
}
|
||
|
||
selectedBusinessSender = item.sender;
|
||
originalBusinessTrack = item.originalTrack || item.sender.track || originalBusinessTrack;
|
||
log(`selected output sender (${reason}): ${item.id}, track=${trackLabel(item.sender.track)}`, true);
|
||
updateSenderSelect();
|
||
updateGateLabel();
|
||
attachAiOutputToBusiness();
|
||
return selectedBusinessSender;
|
||
}
|
||
|
||
async function changeBusinessSender(reason) {
|
||
changingBusinessSender = true;
|
||
try {
|
||
await restoreBusinessAudio();
|
||
selectedBusinessSender = null;
|
||
originalBusinessTrack = null;
|
||
aiAttached = false;
|
||
} finally {
|
||
changingBusinessSender = false;
|
||
}
|
||
senderSelectSignature = '';
|
||
selectBusinessSender(reason);
|
||
if (isAiTakeover) await attachAiOutputToBusiness();
|
||
updateGateLabel();
|
||
}
|
||
|
||
function setVu(id, level) {
|
||
const el = document.getElementById(id);
|
||
if (!el) return;
|
||
el.style.width = `${Math.min(100, level * 2500)}%`;
|
||
el.style.opacity = level > 0 ? '1' : '0.35';
|
||
}
|
||
|
||
function volumePercent(level) {
|
||
if (!Number.isFinite(level) || level < 0.0015) return 0;
|
||
return Math.round(Math.min(100, level * 2500));
|
||
}
|
||
|
||
function volumeDb(level) {
|
||
if (!Number.isFinite(level) || level < 0.0015) return '-inf';
|
||
return `${(20 * Math.log10(level)).toFixed(1)}dB`;
|
||
}
|
||
|
||
function volumeText(level) {
|
||
return `vol=${volumePercent(level)}% ${volumeDb(level)}`;
|
||
}
|
||
|
||
function setVolumeLabel(id, level) {
|
||
const el = document.getElementById(id);
|
||
if (!el) return;
|
||
el.textContent = volumeText(level);
|
||
}
|
||
|
||
function renderVu() {
|
||
refreshAiOutputVuFromAnalyser();
|
||
inputVu *= 0.88;
|
||
outputVu *= 0.88;
|
||
setVu('element2-copilot-input-vu', inputVu);
|
||
setVu('element2-copilot-output-vu', outputVu);
|
||
setVolumeLabel('element2-copilot-input-volume', inputVu);
|
||
setVolumeLabel('element2-copilot-output-volume', outputVu);
|
||
requestAnimationFrame(renderVu);
|
||
}
|
||
|
||
function speechVuLevel(level) {
|
||
const noiseFloor = 0.0015;
|
||
if (!Number.isFinite(level) || level < noiseFloor) return 0;
|
||
return Math.min(0.08, (level - noiseFloor) * 3.2);
|
||
}
|
||
|
||
function analyserVuLevel(analyser, data) {
|
||
if (!analyser || !data) return 0;
|
||
analyser.getByteTimeDomainData(data);
|
||
let sum = 0;
|
||
for (const value of data) {
|
||
const centered = (value - 128) / 128;
|
||
sum += centered * centered;
|
||
}
|
||
return speechVuLevel(Math.sqrt(sum / data.length));
|
||
}
|
||
|
||
function refreshAiOutputVuFromAnalyser() {
|
||
outputVu = Math.max(outputVu, analyserVuLevel(aiOutputAnalyser, aiOutputAnalyserData));
|
||
}
|
||
|
||
function createAudioContext() {
|
||
const AudioContextCtor = window.AudioContext || window.webkitAudioContext;
|
||
return AudioContextCtor ? new AudioContextCtor() : null;
|
||
}
|
||
|
||
function detachAiOutputMeter() {
|
||
try {
|
||
aiOutputAnalyserSource?.disconnect();
|
||
} catch {}
|
||
aiOutputAnalyser = null;
|
||
aiOutputAnalyserData = null;
|
||
aiOutputAnalyserSource = null;
|
||
aiOutputAnalyserTrack = null;
|
||
outputVu = 0;
|
||
}
|
||
|
||
async function attachAiOutputMeter(track) {
|
||
if (!isLive(track)) {
|
||
detachAiOutputMeter();
|
||
return false;
|
||
}
|
||
if (aiOutputAnalyserTrack === track && aiOutputAnalyser) return true;
|
||
detachAiOutputMeter();
|
||
if (!aiOutputAudioContext) {
|
||
aiOutputAudioContext = createAudioContext();
|
||
if (!aiOutputAudioContext) {
|
||
appendChatLine('音频', '当前浏览器不支持 AudioContext，无法显示 AI 输出音量', true);
|
||
return false;
|
||
}
|
||
}
|
||
if (aiOutputAudioContext.state === 'suspended') {
|
||
await aiOutputAudioContext.resume().catch(() => {});
|
||
}
|
||
try {
|
||
const stream = new MediaStream([track]);
|
||
aiOutputAnalyserSource = aiOutputAudioContext.createMediaStreamSource(stream);
|
||
aiOutputAnalyser = aiOutputAudioContext.createAnalyser();
|
||
aiOutputAnalyser.fftSize = 512;
|
||
aiOutputAnalyser.smoothingTimeConstant = 0.55;
|
||
aiOutputAnalyserData = new Uint8Array(aiOutputAnalyser.fftSize);
|
||
aiOutputAnalyserSource.connect(aiOutputAnalyser);
|
||
aiOutputAnalyserTrack = track;
|
||
return true;
|
||
} catch (error) {
|
||
log(`failed to attach AI output meter: ${error.message || error}`);
|
||
detachAiOutputMeter();
|
||
return false;
|
||
}
|
||
}
|
||
|
||
async function refreshStats() {
|
||
await refreshInputStats();
|
||
await refreshOutputStats();
|
||
}
|
||
|
||
async function refreshInputStats() {
|
||
for (const item of remoteAudio.values()) {
|
||
if (!item.receiver?.getStats) continue;
|
||
const report = await item.receiver.getStats();
|
||
report.forEach((stat) => {
|
||
const kind = stat.kind || stat.mediaType;
|
||
if (stat.type !== 'inbound-rtp' || kind !== 'audio') return;
|
||
const prev = inboundMemory.get(stat.id);
|
||
const bytes = stat.bytesReceived || 0;
|
||
const packets = stat.packetsReceived || 0;
|
||
let audioLevel = typeof stat.audioLevel === 'number' ? stat.audioLevel : 0;
|
||
if (
|
||
prev
|
||
&& typeof stat.totalAudioEnergy === 'number'
|
||
&& typeof stat.totalSamplesDuration === 'number'
|
||
&& typeof prev.energy === 'number'
|
||
&& typeof prev.duration === 'number'
|
||
) {
|
||
const energyDelta = stat.totalAudioEnergy - prev.energy;
|
||
const durationDelta = stat.totalSamplesDuration - prev.duration;
|
||
if (energyDelta > 0 && durationDelta > 0) {
|
||
audioLevel = Math.max(audioLevel, Math.sqrt(energyDelta / durationDelta));
|
||
}
|
||
}
|
||
item.bytesReceived = bytes;
|
||
item.packetsReceived = packets;
|
||
item.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0;
|
||
item.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0;
|
||
item.audioLevel = audioLevel;
|
||
inboundMemory.set(stat.id, {
|
||
bytes,
|
||
packets,
|
||
energy: stat.totalAudioEnergy,
|
||
duration: stat.totalSamplesDuration,
|
||
});
|
||
if (item.track === selectedRemoteAudio) {
|
||
inputVu = Math.max(inputVu, speechVuLevel(audioLevel));
|
||
}
|
||
});
|
||
}
|
||
selectRemoteAudio('stats');
|
||
}
|
||
|
||
async function refreshOutputStats() {
|
||
for (const item of businessSenders.values()) {
|
||
if (!item.sender?.getStats) continue;
|
||
item.track = item.sender.track || item.track;
|
||
const report = await item.sender.getStats();
|
||
report.forEach((stat) => {
|
||
const kind = stat.kind || stat.mediaType;
|
||
if (stat.type !== 'outbound-rtp' || kind !== 'audio') return;
|
||
const key = `${item.id}:${stat.id}`;
|
||
const prev = outboundMemory.get(key);
|
||
const bytes = stat.bytesSent || 0;
|
||
const packets = stat.packetsSent || 0;
|
||
item.bytesSent = bytes;
|
||
item.packetsSent = packets;
|
||
item.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0;
|
||
item.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0;
|
||
outboundMemory.set(key, { bytes, packets });
|
||
});
|
||
}
|
||
selectBusinessSender('stats');
|
||
updateSenderSelect();
|
||
}
|
||
|
||
async function outboundBytes(sender) {
|
||
if (!sender?.getStats) return null;
|
||
let total = 0;
|
||
const report = await sender.getStats();
|
||
report.forEach((stat) => {
|
||
const kind = stat.kind || stat.mediaType;
|
||
if (stat.type === 'outbound-rtp' && kind === 'audio') total += stat.bytesSent || 0;
|
||
});
|
||
return total;
|
||
}
|
||
|
||
function aiState() {
|
||
if (!aiPc) return 'pc=null';
|
||
return `conn=${aiPc.connectionState},ice=${aiPc.iceConnectionState},signaling=${aiPc.signalingState},remote=${aiPc.remoteDescription?.type || 'none'}`;
|
||
}
|
||
|
||
function aiReady() {
|
||
return Boolean(aiPc?.remoteDescription && (
|
||
aiPc.connectionState === 'connected'
|
||
|| aiPc.iceConnectionState === 'connected'
|
||
|| aiPc.iceConnectionState === 'completed'
|
||
));
|
||
}
|
||
|
||
function waitForAiReady(timeoutMs = 10000) {
|
||
if (aiReady()) return Promise.resolve(true);
|
||
return new Promise((resolve) => {
|
||
const startedAt = Date.now();
|
||
const timer = setInterval(() => {
|
||
if (aiReady()) {
|
||
clearInterval(timer);
|
||
resolve(true);
|
||
} else if (Date.now() - startedAt >= timeoutMs) {
|
||
clearInterval(timer);
|
||
resolve(false);
|
||
}
|
||
}, 250);
|
||
});
|
||
}
|
||
|
||
function checkAiInput(track) {
|
||
if (inputHealthTimer) clearTimeout(inputHealthTimer);
|
||
inputHealthTimer = setTimeout(async () => {
|
||
if (!aiAudioSender || aiAudioSender.track !== track) return;
|
||
await waitForAiReady();
|
||
const before = await outboundBytes(aiAudioSender);
|
||
await new Promise((resolve) => setTimeout(resolve, 2500));
|
||
const after = await outboundBytes(aiAudioSender);
|
||
log(`AI input packet check: before=${before}, after=${after}, ${aiState()}`, true);
|
||
}, 500);
|
||
}
|
||
|
||
function checkAiOutput(track) {
|
||
if (outputHealthTimer) clearTimeout(outputHealthTimer);
|
||
outputHealthTimer = setTimeout(async () => {
|
||
if (!selectedBusinessSender || selectedBusinessSender.track !== track) return;
|
||
const before = await outboundBytes(selectedBusinessSender);
|
||
await new Promise((resolve) => setTimeout(resolve, 2500));
|
||
const after = await outboundBytes(selectedBusinessSender);
|
||
log(`AI output packet check: before=${before}, after=${after}, sender=${trackLabel(selectedBusinessSender.track)}`, true);
|
||
}, 800);
|
||
}
|
||
|
||
async function attachAudioToAi() {
|
||
if (!shouldSendAudioToAi()) return false;
|
||
if (!aiAudioSender || !isLive(selectedRemoteAudio)) return false;
|
||
if (aiAudioSender.track === selectedRemoteAudio) return true;
|
||
await aiAudioSender.replaceTrack(selectedRemoteAudio);
|
||
log(`AI input uses ${trackLabel(selectedRemoteAudio)} from ${STATIC_AUDIO_ELEMENT_REASON}`, true);
|
||
checkAiInput(selectedRemoteAudio);
|
||
return true;
|
||
}
|
||
|
||
async function attachVideoToAi() {
|
||
if (!shouldSendVideoToAi()) return false;
|
||
if (!aiVideoSender || !isLive(selectedRemoteVideo)) return false;
|
||
if (aiVideoSender.track === selectedRemoteVideo) return true;
|
||
await aiVideoSender.replaceTrack(selectedRemoteVideo);
|
||
log(`AI video uses ${trackLabel(selectedRemoteVideo)} from ${STATIC_VIDEO_ELEMENT_REASON}`, true);
|
||
return true;
|
||
}
|
||
|
||
async function attachAiOutputToBusiness() {
|
||
selectBusinessSender('before-output');
|
||
if (!isAiTakeover || !selectedBusinessSender || !isLive(aiAudioTrack)) {
|
||
log(`AI output waiting: takeover=${isAiTakeover}, sender=${Boolean(selectedBusinessSender)}, aiTrack=${trackLabel(aiAudioTrack)}`);
|
||
return false;
|
||
}
|
||
if (selectedBusinessSender.track === aiAudioTrack) return true;
|
||
await selectedBusinessSender.replaceTrack(aiAudioTrack);
|
||
aiAttached = true;
|
||
log(`business sender uses AI audio ${trackLabel(aiAudioTrack)}`, true);
|
||
checkAiOutput(aiAudioTrack);
|
||
return true;
|
||
}
|
||
|
||
async function restoreBusinessAudio() {
|
||
if (!selectedBusinessSender || !aiAttached) return;
|
||
await selectedBusinessSender.replaceTrack(originalBusinessTrack || null);
|
||
aiAttached = false;
|
||
log(`business sender restored ${trackLabel(originalBusinessTrack)}`, true);
|
||
}
|
||
|
||
function wrapTrackListener(listener) {
|
||
if (!listener || listener.__aiElement2GatedWrapped) return listener;
|
||
const wrapped = function (event) {
|
||
const ownerPc = this instanceof RTCPeerConnection ? this : event?.currentTarget;
|
||
if (!isInternalPc(ownerPc) && event?.track) {
|
||
rememberRemoteReceiver(event.receiver, 'pc-track');
|
||
for (const stream of event.streams || []) rememberRemoteStream(stream, 'pc-track-stream');
|
||
}
|
||
if (typeof listener === 'function') return listener.apply(this, arguments);
|
||
if (listener?.handleEvent) return listener.handleEvent(event);
|
||
return undefined;
|
||
};
|
||
wrapped.__aiElement2GatedWrapped = true;
|
||
return wrapped;
|
||
}
|
||
|
||
RTCPeerConnection.prototype.addEventListener = function (type, listener, options) {
|
||
if (type === 'track') return originalAddEventListener.call(this, type, wrapTrackListener(listener), options);
|
||
return originalAddEventListener.call(this, type, listener, options);
|
||
};
|
||
|
||
try {
|
||
const descriptor = Object.getOwnPropertyDescriptor(RTCPeerConnection.prototype, 'ontrack');
|
||
Object.defineProperty(RTCPeerConnection.prototype, 'ontrack', {
|
||
configurable: true,
|
||
enumerable: true,
|
||
get() {
|
||
return descriptor?.get ? descriptor.get.call(this) : this.__aiElement2GatedOnTrack || null;
|
||
},
|
||
set(listener) {
|
||
const wrapped = wrapTrackListener(listener);
|
||
this.__aiElement2GatedOnTrack = wrapped;
|
||
if (descriptor?.set) descriptor.set.call(this, wrapped);
|
||
else this.addEventListener('track', wrapped);
|
||
},
|
||
});
|
||
} catch (error) {
|
||
console.warn('[AI Element2 Gated] failed to wrap ontrack', error);
|
||
}
|
||
|
||
RTCPeerConnection.prototype.addTrack = function (track, ...streams) {
|
||
const sender = originalAddTrack.call(this, track, ...streams);
|
||
if (!isInternalPc(this) && track?.kind === 'audio') rememberBusinessSender(sender, track, 'pc-addTrack');
|
||
return sender;
|
||
};
|
||
|
||
if (originalAddTransceiver) {
|
||
RTCPeerConnection.prototype.addTransceiver = function (trackOrKind, init) {
|
||
const transceiver = originalAddTransceiver.call(this, trackOrKind, init);
|
||
if (!isInternalPc(this)) {
|
||
const kind = trackOrKind instanceof MediaStreamTrack ? trackOrKind.kind : trackOrKind;
|
||
if (kind === 'audio') {
|
||
const track = trackOrKind instanceof MediaStreamTrack ? trackOrKind : transceiver.sender?.track;
|
||
rememberBusinessSender(transceiver.sender, track, 'pc-addTransceiver');
|
||
}
|
||
}
|
||
return transceiver;
|
||
};
|
||
}
|
||
|
||
RTCPeerConnection.prototype.setRemoteDescription = function () {
|
||
const result = originalSetRemoteDescription.apply(this, arguments);
|
||
if (!isInternalPc(this)) {
|
||
Promise.resolve(result).then(() => {
|
||
for (const receiver of this.getReceivers?.() || []) rememberRemoteReceiver(receiver, 'receiver-after-srd');
|
||
for (const sender of this.getSenders?.() || []) rememberBusinessSender(sender, sender.track, 'sender-after-srd');
|
||
}).catch(() => {});
|
||
}
|
||
return result;
|
||
};
|
||
|
||
if (mediaSrcObjectDescriptor?.set) {
|
||
Object.defineProperty(HTMLMediaElement.prototype, 'srcObject', {
|
||
configurable: true,
|
||
enumerable: mediaSrcObjectDescriptor.enumerable,
|
||
get() {
|
||
return mediaSrcObjectDescriptor.get.call(this);
|
||
},
|
||
set(stream) {
|
||
if (!this.__aiElement2GatedCopilotElement) {
|
||
const reason = elementReasonFor(this) || `${this.tagName.toLowerCase()}-srcObject`;
|
||
rememberRemoteStream(stream, reason);
|
||
}
|
||
return mediaSrcObjectDescriptor.set.call(this, stream);
|
||
},
|
||
});
|
||
}
|
||
|
||
function scanMediaElements() {
|
||
document.querySelectorAll('audio').forEach((element, index) => {
|
||
if (element.__aiElement2GatedCopilotElement) return;
|
||
rememberRemoteStream(element.srcObject, `element-${index}:audio`);
|
||
});
|
||
document.querySelectorAll('video').forEach((element, index) => {
|
||
if (element.__aiElement2GatedCopilotElement) return;
|
||
rememberRemoteStream(element.srcObject, `element-${index}:video`);
|
||
});
|
||
selectRemoteAudio('media-scan');
|
||
selectRemoteVideo('media-scan');
|
||
updateGateLabel();
|
||
}
|
||
|
||
setInterval(scanMediaElements, 1000);
|
||
|
||
function endpointUrl(baseUrl, path) {
|
||
const normalizedBaseUrl = baseUrl.trim().replace(/\/+$/, '');
|
||
return new URL(`${normalizedBaseUrl}${path}`).toString();
|
||
}
|
||
|
||
function basicAuthorization(username, password) {
|
||
const bytes = new TextEncoder().encode(`${username}:${password}`);
|
||
let binary = '';
|
||
bytes.forEach((byte) => { binary += String.fromCharCode(byte); });
|
||
return `Basic ${btoa(binary)}`;
|
||
}
|
||
|
||
function aiRequestHeaders(extra = {}) {
|
||
return {
|
||
...extra,
|
||
...(aiAuthorization ? { Authorization: aiAuthorization } : {}),
|
||
};
|
||
}
|
||
|
||
async function fetchIceServers(baseUrl) {
|
||
try {
|
||
const iceUrl = endpointUrl(baseUrl, '/api/webrtc/ice-servers');
|
||
const response = await fetch(iceUrl, {
|
||
credentials: 'include',
|
||
headers: aiRequestHeaders(),
|
||
});
|
||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||
const payload = await response.json();
|
||
if (Array.isArray(payload.iceServers)) return payload.iceServers;
|
||
} catch (error) {
|
||
log(`ICE server config unavailable, using public STUN: ${error.message || error}`);
|
||
}
|
||
return [{ urls: 'stun:stun.l.google.com:19302' }];
|
||
}
|
||
|
||
function queueAiIceCandidate(candidate) {
|
||
if (!candidate) return;
|
||
aiCandidateQueue.push({
|
||
candidate: candidate.candidate,
|
||
sdp_mid: candidate.sdpMid,
|
||
sdp_mline_index: candidate.sdpMLineIndex,
|
||
});
|
||
if (!aiCandidateFlushTimer) {
|
||
aiCandidateFlushTimer = setTimeout(flushAiIceCandidates, 200);
|
||
}
|
||
}
|
||
|
||
async function flushAiIceCandidates() {
|
||
aiCandidateFlushTimer = null;
|
||
if (!aiOfferUrl || !aiPcId || !aiCanSendCandidates || aiCandidateQueue.length === 0) return;
|
||
const candidates = aiCandidateQueue.splice(0, aiCandidateQueue.length);
|
||
try {
|
||
const response = await fetch(aiOfferUrl, {
|
||
method: 'PATCH',
|
||
credentials: 'include',
|
||
headers: aiRequestHeaders({ 'Content-Type': 'application/json' }),
|
||
body: JSON.stringify({ pc_id: aiPcId, candidates }),
|
||
});
|
||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||
} catch (error) {
|
||
aiCandidateQueue.unshift(...candidates);
|
||
log(`failed to send ICE candidates: ${error.message || error}`);
|
||
}
|
||
}
|
||
|
||
function sendAiSignalling(message) {
|
||
if (aiDc?.readyState !== 'open') return;
|
||
aiDc.send(JSON.stringify({ type: 'signalling', message }));
|
||
}
|
||
|
||
async function negotiateAi(restartPc = false) {
|
||
if (!aiPc || !aiOfferUrl || !aiAssistantId || aiNegotiating) return false;
|
||
aiNegotiating = true;
|
||
try {
|
||
const offer = await aiPc.createOffer();
|
||
await aiPc.setLocalDescription(offer);
|
||
const response = await fetch(aiOfferUrl, {
|
||
method: 'POST',
|
||
credentials: 'include',
|
||
headers: aiRequestHeaders({
|
||
'Content-Type': 'application/json',
|
||
'X-Pipecat-Assistant-ID': aiAssistantId,
|
||
}),
|
||
body: JSON.stringify({
|
||
pc_id: aiPcId,
|
||
sdp: aiPc.localDescription.sdp,
|
||
type: aiPc.localDescription.type,
|
||
restart_pc: restartPc,
|
||
requestData: {
|
||
assistant_id: aiAssistantId,
|
||
vision_enabled: shouldSendVideoToAi(),
|
||
dynamic_variables: {},
|
||
},
|
||
}),
|
||
});
|
||
if (!response.ok) {
|
||
const detail = await response.text().catch(() => '');
|
||
throw new Error(`HTTP ${response.status}${detail ? `: ${detail}` : ''}`);
|
||
}
|
||
const answer = await response.json();
|
||
aiPcId = answer.pc_id;
|
||
await aiPc.setRemoteDescription({ type: answer.type, sdp: answer.sdp });
|
||
aiCanSendCandidates = true;
|
||
await flushAiIceCandidates();
|
||
log(`AI WebRTC answer applied; ${aiState()}`, true);
|
||
return true;
|
||
} finally {
|
||
aiNegotiating = false;
|
||
}
|
||
}
|
||
|
||
async function handleAiSignalling(message) {
|
||
try {
|
||
if (message?.type === 'renegotiate') {
|
||
await negotiateAi(false);
|
||
} else if (message?.type === 'peerLeft') {
|
||
setStatus('disconnected');
|
||
log('AI peer left', true);
|
||
} else {
|
||
log(`unknown SmallWebRTC signalling: ${message?.type || 'missing type'}`);
|
||
}
|
||
} catch (error) {
|
||
log(`SmallWebRTC renegotiation failed: ${error.message || error}`, true);
|
||
}
|
||
}
|
||
|
||
async function connectAi(baseUrl, assistantId, username, password) {
|
||
scanMediaElements();
|
||
selectRemoteAudio('connect');
|
||
selectRemoteVideo('connect');
|
||
selectBusinessSender('connect');
|
||
const status = gateStatus();
|
||
updateGateLabel();
|
||
if (!status.operatorOnline || !status.participantOnline || !status.senderReady) {
|
||
appendChatLine('门槛', gateMessage(status), true);
|
||
setStatus('disconnected');
|
||
return false;
|
||
}
|
||
|
||
disconnectAi();
|
||
selectedRemoteAudio = status.track;
|
||
|
||
try {
|
||
aiOfferUrl = endpointUrl(baseUrl, '/api/webrtc/offer');
|
||
} catch {
|
||
appendChatLine('配置', 'Base URL 格式无效，请填写包含 http:// 或 https:// 的完整地址', true);
|
||
setStatus('failed');
|
||
return false;
|
||
}
|
||
aiAssistantId = assistantId;
|
||
aiAuthorization = basicAuthorization(username, password);
|
||
aiPcId = pcId();
|
||
aiCandidateQueue = [];
|
||
aiCanSendCandidates = false;
|
||
const iceServers = await fetchIceServers(baseUrl);
|
||
aiPc = new RTCPeerConnection({ iceServers });
|
||
markInternalPc(aiPc);
|
||
aiPc.onconnectionstatechange = () => log(`AI PC connectionState=${aiPc.connectionState}`);
|
||
aiPc.oniceconnectionstatechange = () => log(`AI PC iceConnectionState=${aiPc.iceConnectionState}`);
|
||
|
||
aiDc = aiPc.createDataChannel('chat');
|
||
aiDc.onopen = () => {
|
||
setStatus('connected');
|
||
log('AI data channel open', true);
|
||
sendAiSignalling({ type: 'trackStatus', receiver_index: 0, enabled: shouldSendAudioToAi() });
|
||
sendAiSignalling({ type: 'trackStatus', receiver_index: 1, enabled: shouldSendVideoToAi() });
|
||
aiDc.send(JSON.stringify({ type: 'client-ready' }));
|
||
aiKeepAliveTimer = setInterval(() => {
|
||
if (aiDc?.readyState === 'open') aiDc.send(`ping: ${Date.now()}`);
|
||
}, 1000);
|
||
};
|
||
aiDc.onclose = () => setStatus('disconnected');
|
||
aiDc.onmessage = (event) => {
|
||
try {
|
||
const msg = JSON.parse(event.data);
|
||
const content = msg.delta || msg.content;
|
||
if (msg.type === 'signalling') {
|
||
void handleAiSignalling(msg.message);
|
||
} else if (msg.type === 'transcript') {
|
||
logUserTranscript(content);
|
||
} else if (msg.type === 'assistant-text-start') {
|
||
startAssistantLine();
|
||
} else if (msg.type === 'assistant-text-delta') {
|
||
appendAssistantDelta(content);
|
||
} else if (msg.type === 'assistant-text-end') {
|
||
finishAssistantLine();
|
||
} else {
|
||
log(`data channel message: ${msg.type}`);
|
||
}
|
||
} catch {
|
||
appendChatLine('DATA', event.data);
|
||
}
|
||
};
|
||
|
||
const sendAudio = shouldSendAudioToAi();
|
||
const audioTransceiver = aiPc.addTransceiver('audio', { direction: 'sendrecv' });
|
||
aiAudioSender = audioTransceiver.sender;
|
||
if (sendAudio && isLive(selectedRemoteAudio)) await aiAudioSender.replaceTrack(selectedRemoteAudio);
|
||
log(!sendAudio
|
||
? 'AI audio uplink disabled by panel setting'
|
||
: isLive(selectedRemoteAudio)
|
||
? `AI input starts with ${trackLabel(selectedRemoteAudio)} (${STATIC_AUDIO_ELEMENT_REASON})`
|
||
: `AI input waiting for ${STATIC_AUDIO_ELEMENT_REASON}`, true);
|
||
|
||
const sendVideo = shouldSendVideoToAi();
|
||
const shouldSendVideo = sendVideo && isLive(selectedRemoteVideo) && !selectedRemoteVideo.muted;
|
||
const videoTransceiver = aiPc.addTransceiver('video', { direction: 'sendrecv' });
|
||
aiVideoSender = videoTransceiver.sender;
|
||
if (shouldSendVideo) {
|
||
await aiVideoSender.replaceTrack(selectedRemoteVideo);
|
||
log(`AI video starts with ${trackLabel(selectedRemoteVideo)} (${STATIC_VIDEO_ELEMENT_REASON})`);
|
||
} else {
|
||
log(sendVideo
|
||
? `AI video disabled: waiting for live track at ${STATIC_VIDEO_ELEMENT_REASON}`
|
||
: 'AI video uplink disabled by panel setting');
|
||
}
|
||
|
||
aiPc.ontrack = (event) => {
|
||
if (event.track.kind !== 'audio') return;
|
||
aiOutputTrackIds.add(event.track.id);
|
||
remoteAudio.delete(event.track.id);
|
||
aiAudioTrack = event.track;
|
||
attachAiOutputMeter(event.track);
|
||
log(`received AI audio ${trackLabel(aiAudioTrack)}`, true);
|
||
aiAudioTrack.onunmute = () => {
|
||
log(`AI audio unmuted ${trackLabel(aiAudioTrack)}`);
|
||
attachAiOutputToBusiness();
|
||
};
|
||
attachAiOutputToBusiness();
|
||
};
|
||
|
||
aiPc.onicecandidate = (event) => {
|
||
queueAiIceCandidate(event.candidate);
|
||
};
|
||
|
||
setStatus('connecting');
|
||
try {
|
||
await negotiateAi(false);
|
||
log('sent SmallWebRTC HTTP offer', true);
|
||
} catch (error) {
|
||
log(`AI HTTP signalling error: ${error.message || error}`, true);
|
||
disconnectAi();
|
||
setStatus('failed');
|
||
return false;
|
||
}
|
||
|
||
attachAudioToAi();
|
||
attachVideoToAi();
|
||
return true;
|
||
}
|
||
|
||
function disconnectAi() {
|
||
try { aiDc?.close(); } catch {}
|
||
try { aiPc?.close(); } catch {}
|
||
if (aiCandidateFlushTimer) clearTimeout(aiCandidateFlushTimer);
|
||
if (aiKeepAliveTimer) clearInterval(aiKeepAliveTimer);
|
||
aiDc = null;
|
||
aiPc = null;
|
||
aiPcId = null;
|
||
aiOfferUrl = null;
|
||
aiAssistantId = null;
|
||
aiAuthorization = null;
|
||
aiCandidateQueue = [];
|
||
aiCandidateFlushTimer = null;
|
||
aiKeepAliveTimer = null;
|
||
aiNegotiating = false;
|
||
aiCanSendCandidates = false;
|
||
aiAudioSender = null;
|
||
aiVideoSender = null;
|
||
aiAudioTrack = null;
|
||
detachAiOutputMeter();
|
||
if (inputRecorder?.state === 'recording') stopInputRecording();
|
||
if (senderRecorder?.state === 'recording') senderRecorder.stop();
|
||
if (videoRecorder?.state === 'recording') stopVideoRecording();
|
||
setStatus('disconnected');
|
||
}
|
||
|
||
function setStatus(status) {
|
||
const el = document.getElementById('element2-copilot-status');
|
||
if (!el) return;
|
||
el.textContent = {
|
||
disconnected: '未连接',
|
||
connecting: '连接中',
|
||
connected: '已连接',
|
||
failed: '连接失败',
|
||
}[status] || status;
|
||
}
|
||
|
||
function updateModeButton() {
|
||
const button = document.getElementById('element2-copilot-toggle');
|
||
if (!button) return;
|
||
if (isAiTakeover) {
|
||
button.textContent = '当前：AI 接管｜点击切回人工';
|
||
button.style.background = '#8b5cf6';
|
||
} else {
|
||
button.textContent = '当前：人工接管｜点击切换 AI';
|
||
button.style.background = '#10b981';
|
||
}
|
||
}
|
||
|
||
window.__aiElement2GatedDump = function () {
|
||
const status = gateStatus();
|
||
return {
|
||
audioSelection: {
|
||
mode: audioSelectionMode,
|
||
manualTrackId: manualAudioTrackId,
|
||
preferredElement: STATIC_AUDIO_ELEMENT_REASON,
|
||
},
|
||
videoSelection: {
|
||
mode: videoSelectionMode,
|
||
manualTrackId: manualVideoTrackId,
|
||
preferredElement: STATIC_VIDEO_ELEMENT_REASON,
|
||
},
|
||
senderSelection: {
|
||
mode: senderSelectionMode,
|
||
manualSenderId,
|
||
},
|
||
staticVideoElement: STATIC_VIDEO_ELEMENT_REASON,
|
||
gate: {
|
||
bothParties: status.bothParties,
|
||
participantOnline: status.participantOnline,
|
||
operatorOnline: status.operatorOnline,
|
||
senderReady: status.senderReady,
|
||
track: trackLabel(status.track),
|
||
},
|
||
gateMessage: gateMessage(status),
|
||
selectedInput: trackLabel(selectedRemoteAudio),
|
||
selectedVideo: trackLabel(selectedRemoteVideo),
|
||
aiVideoSenderTrack: trackLabel(aiVideoSender?.track),
|
||
selectedOutput: trackLabel(selectedBusinessSender?.track),
|
||
originalOutput: trackLabel(originalBusinessTrack),
|
||
aiAudio: trackLabel(aiAudioTrack),
|
||
aiAttached,
|
||
aiState: aiState(),
|
||
recording: inputRecorder?.state === 'recording' ? {
|
||
track: trackLabel(inputRecordingTrack),
|
||
seconds: ((Date.now() - inputRecordingStartedAt) / 1000).toFixed(1),
|
||
chunks: inputRecordingChunks.length,
|
||
} : null,
|
||
senderRecording: senderRecorder?.state === 'recording' ? {
|
||
track: trackLabel(senderRecordingTrack),
|
||
seconds: ((Date.now() - senderRecordingStartedAt) / 1000).toFixed(1),
|
||
chunks: senderRecordingChunks.length,
|
||
} : null,
|
||
videoRecording: videoRecorder?.state === 'recording' ? {
|
||
track: trackLabel(videoRecordingTrack),
|
||
seconds: ((Date.now() - videoRecordingStartedAt) / 1000).toFixed(1),
|
||
chunks: videoRecordingChunks.length,
|
||
} : null,
|
||
inputs: Array.from(remoteAudio.values()).map((item) => ({
|
||
selected: item.track === selectedRemoteAudio,
|
||
id: item.track.id,
|
||
muted: item.track.muted,
|
||
live: item.track.readyState,
|
||
hasReceiver: Boolean(item.receiver),
|
||
source: audioSourceLabel(item),
|
||
bytesReceived: item.bytesReceived || 0,
|
||
packetsReceived: item.packetsReceived || 0,
|
||
bytesDelta: item.bytesDelta,
|
||
packetsDelta: item.packetsDelta,
|
||
audioLevel: item.audioLevel || 0,
|
||
audioScore: Math.round(item.audioScore || scoreAudioCandidate(item) || 0),
|
||
matchesStaticElement: hasElementReason(item, STATIC_AUDIO_ELEMENT_REASON),
|
||
reasons: Array.from(item.reasons),
|
||
})),
|
||
videos: Array.from(remoteVideo.values()).map((item) => ({
|
||
selected: item.track === selectedRemoteVideo,
|
||
id: item.track.id,
|
||
muted: item.track.muted,
|
||
live: item.track.readyState,
|
||
hasReceiver: Boolean(item.receiver),
|
||
matchesStaticElement: hasElementReason(item, STATIC_VIDEO_ELEMENT_REASON),
|
||
reasons: Array.from(item.reasons),
|
||
})),
|
||
outputs: Array.from(businessSenders.values()).map((item) => ({
|
||
selected: item.sender === selectedBusinessSender,
|
||
id: item.id,
|
||
track: trackLabel(item.sender.track || item.track),
|
||
originalTrack: trackLabel(item.originalTrack),
|
||
pcAddTrack: item.sources.has('pc-addTrack'),
|
||
isRemoteInput: isRemoteInputTrack(item.sender.track || item.track),
|
||
bytesSent: item.bytesSent,
|
||
packetsSent: item.packetsSent,
|
||
bytesDelta: item.bytesDelta,
|
||
packetsDelta: item.packetsDelta,
|
||
sources: Array.from(item.sources),
|
||
})),
|
||
};
|
||
};
|
||
|
||
function renderPanel() {
|
||
if (!document.body || document.getElementById('element2-copilot-panel')) return;
|
||
const panel = document.createElement('div');
|
||
panel.id = 'element2-copilot-panel';
|
||
panel.style.cssText = [
|
||
'position:fixed',
|
||
'top:20px',
|
||
'left:20px',
|
||
'width:440px',
|
||
'max-height:calc(100vh - 40px)',
|
||
'overflow:auto',
|
||
'z-index:999999',
|
||
'box-sizing:border-box',
|
||
'padding:16px',
|
||
'border-radius:10px',
|
||
'background:rgba(17,24,39,.96)',
|
||
'color:white',
|
||
'font-family:system-ui,-apple-system,BlinkMacSystemFont,sans-serif',
|
||
'box-shadow:0 12px 32px rgba(0,0,0,.35)',
|
||
].join(';');
|
||
panel.innerHTML = `
|
||
<div style="display:flex;justify-content:space-between;gap:12px;align-items:center;margin-bottom:12px;">
|
||
<strong style="font-size:15px;color:#f9a8d4;">AI 托管</strong>
|
||
<span id="element2-copilot-status" style="font-size:12px;color:#d1d5db;">未连接</span>
|
||
</div>
|
||
<div style="font-size:11px;color:#9ca3af;margin-bottom:10px;line-height:1.5;">
|
||
<span id="element2-copilot-gate" style="color:#fcd34d;">等待当事人在线</span>
|
||
</div>
|
||
<div style="display:grid;grid-template-columns:1.35fr 1fr;gap:8px;margin-bottom:10px;">
|
||
<div>
|
||
<label style="display:block;font-size:12px;color:#9ca3af;margin-bottom:4px;">AI 服务 Base URL</label>
|
||
<input id="element2-copilot-url" value="${DEFAULT_API_BASE_URL}" placeholder="例如：https://ai.example.com" style="width:100%;box-sizing:border-box;padding:8px;border:1px solid #4b5563;border-radius:6px;background:#111827;color:white;">
|
||
</div>
|
||
<div>
|
||
<label style="display:block;font-size:12px;color:#9ca3af;margin-bottom:4px;">助手 ID</label>
|
||
<input id="element2-copilot-assistant" placeholder="请输入助手 ID" style="width:100%;box-sizing:border-box;padding:8px;border:1px solid #4b5563;border-radius:6px;background:#111827;color:white;">
|
||
</div>
|
||
</div>
|
||
<div style="display:grid;grid-template-columns:1fr 1fr;gap:8px;margin-bottom:12px;">
|
||
<div>
|
||
<label style="display:block;font-size:12px;color:#9ca3af;margin-bottom:4px;">管理员用户名</label>
|
||
<input id="element2-copilot-username" value="admin" autocomplete="username" style="width:100%;box-sizing:border-box;padding:8px;border:1px solid #4b5563;border-radius:6px;background:#111827;color:white;">
|
||
</div>
|
||
<div>
|
||
<label style="display:block;font-size:12px;color:#9ca3af;margin-bottom:4px;">管理员密码</label>
|
||
<input id="element2-copilot-password" type="password" autocomplete="current-password" placeholder="请输入密码" style="width:100%;box-sizing:border-box;padding:8px;border:1px solid #4b5563;border-radius:6px;background:#111827;color:white;">
|
||
</div>
|
||
</div>
|
||
<div style="margin-bottom:12px;padding:10px;border:1px solid #374151;border-radius:8px;background:rgba(3,7,18,.45);">
|
||
<div style="display:grid;grid-template-columns:110px 1fr 34px;gap:8px;align-items:end;margin-bottom:9px;">
|
||
<div>
|
||
<label style="display:block;font-size:12px;color:#9ca3af;margin-bottom:4px;">AI 输入选轨</label>
|
||
<select id="element2-copilot-audio-mode" style="width:100%;box-sizing:border-box;padding:8px;border:1px solid #4b5563;border-radius:6px;background:#111827;color:white;">
|
||
<option value="auto">自动选轨</option>
|
||
<option value="manual">手动选轨</option>
|
||
<option value="off">不传音频</option>
|
||
</select>
|
||
</div>
|
||
<div>
|
||
<label style="display:block;font-size:12px;color:#9ca3af;margin-bottom:4px;">远端音轨</label>
|
||
<select id="element2-copilot-audio-track" style="width:100%;box-sizing:border-box;padding:8px;border:1px solid #4b5563;border-radius:6px;background:#111827;color:white;">
|
||
<option value="">暂无可用远端音轨</option>
|
||
</select>
|
||
</div>
|
||
<button id="element2-copilot-record" title="录制 AI 输入" aria-label="录制 AI 输入" style="width:34px;height:34px;padding:0;border:0;border-radius:6px;background:#2563eb;color:white;font-size:13px;line-height:34px;cursor:pointer;">●</button>
|
||
</div>
|
||
<div style="display:flex;justify-content:space-between;gap:8px;font-size:12px;color:#d1fae5;margin-bottom:4px;"><span id="element2-copilot-selected-audio" style="overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">截取给 AI 的音频音量</span><span id="element2-copilot-input-volume" style="flex:0 0 auto;">vol=0% -inf</span></div>
|
||
<div style="height:8px;border-radius:999px;overflow:hidden;background:#374151;"><div id="element2-copilot-input-vu" style="height:100%;width:0%;background:linear-gradient(90deg,#10b981,#fcd34d);transition:width .06s linear;"></div></div>
|
||
</div>
|
||
<div style="margin-bottom:12px;padding:10px;border:1px solid #374151;border-radius:8px;background:rgba(3,7,18,.45);">
|
||
<div style="display:grid;grid-template-columns:110px 1fr 34px;gap:8px;align-items:end;margin-bottom:9px;">
|
||
<div>
|
||
<label style="display:block;font-size:12px;color:#9ca3af;margin-bottom:4px;">AI 输出 sender</label>
|
||
<select id="element2-copilot-sender-mode" style="width:100%;box-sizing:border-box;padding:8px;border:1px solid #4b5563;border-radius:6px;background:#111827;color:white;">
|
||
<option value="auto">自动选择</option>
|
||
<option value="manual">手动选择</option>
|
||
</select>
|
||
</div>
|
||
<div>
|
||
<label style="display:block;font-size:12px;color:#9ca3af;margin-bottom:4px;">业务上行 sender</label>
|
||
<select id="element2-copilot-sender" style="width:100%;box-sizing:border-box;padding:8px;border:1px solid #4b5563;border-radius:6px;background:#111827;color:white;">
|
||
<option value="">暂无可用 sender</option>
|
||
</select>
|
||
</div>
|
||
<button id="element2-copilot-record-sender" title="录制 sender 音轨" aria-label="录制 sender 音轨" style="width:34px;height:34px;padding:0;border:0;border-radius:6px;background:#7c3aed;color:white;font-size:13px;line-height:34px;cursor:pointer;">●</button>
|
||
</div>
|
||
<div style="display:flex;justify-content:space-between;gap:8px;font-size:12px;color:#bfdbfe;margin-bottom:4px;"><span style="overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">AI 发送给当事人的音频音量</span><span id="element2-copilot-output-volume" style="flex:0 0 auto;">vol=0% -inf</span></div>
|
||
<div style="height:8px;border-radius:999px;overflow:hidden;background:#374151;"><div id="element2-copilot-output-vu" style="height:100%;width:0%;background:linear-gradient(90deg,#60a5fa,#c084fc);transition:width .06s linear;"></div></div>
|
||
</div>
|
||
<div style="margin-bottom:12px;padding:10px;border:1px solid #374151;border-radius:8px;background:rgba(3,7,18,.45);">
|
||
<div style="display:grid;grid-template-columns:110px 1fr 34px;gap:8px;align-items:end;">
|
||
<div>
|
||
<label style="display:block;font-size:12px;color:#9ca3af;margin-bottom:4px;">AI 视频选轨</label>
|
||
<select id="element2-copilot-video-mode" style="width:100%;box-sizing:border-box;padding:8px;border:1px solid #4b5563;border-radius:6px;background:#111827;color:white;">
|
||
<option value="auto">自动选轨</option>
|
||
<option value="manual">手动选轨</option>
|
||
<option value="off">不传视频</option>
|
||
</select>
|
||
</div>
|
||
<div>
|
||
<label style="display:block;font-size:12px;color:#9ca3af;margin-bottom:4px;">远端视频轨</label>
|
||
<select id="element2-copilot-video-track" style="width:100%;box-sizing:border-box;padding:8px;border:1px solid #4b5563;border-radius:6px;background:#111827;color:white;">
|
||
<option value="">暂无可用远端视频</option>
|
||
</select>
|
||
</div>
|
||
<button id="element2-copilot-record-video" title="录制 AI 视频" aria-label="录制 AI 视频" style="width:34px;height:34px;padding:0;border:0;border-radius:6px;background:#475569;color:white;font-size:13px;line-height:34px;cursor:pointer;">●</button>
|
||
</div>
|
||
</div>
|
||
<button id="element2-copilot-toggle" style="width:100%;padding:12px;border:0;border-radius:6px;background:#10b981;color:white;font-weight:700;cursor:pointer;">当前：人工接管｜点击切换 AI</button>
|
||
<div id="element2-copilot-log" style="height:170px;margin-top:12px;padding:8px;overflow:auto;border:1px solid #374151;border-radius:6px;background:#030712;color:#a7f3d0;font:11px ui-monospace,Menlo,monospace;">
|
||
<span data-placeholder="1" style="color:#6b7280;">等待交互...</span>
|
||
</div>
|
||
`;
|
||
document.body.appendChild(panel);
|
||
|
||
const toggle = document.getElementById('element2-copilot-toggle');
|
||
const recordButton = document.getElementById('element2-copilot-record');
|
||
const senderRecordButton = document.getElementById('element2-copilot-record-sender');
|
||
const videoRecordButton = document.getElementById('element2-copilot-record-video');
|
||
const urlInput = document.getElementById('element2-copilot-url');
|
||
const assistantInput = document.getElementById('element2-copilot-assistant');
|
||
const usernameInput = document.getElementById('element2-copilot-username');
|
||
const passwordInput = document.getElementById('element2-copilot-password');
|
||
const audioModeInput = document.getElementById('element2-copilot-audio-mode');
|
||
const audioTrackInput = document.getElementById('element2-copilot-audio-track');
|
||
const videoModeInput = document.getElementById('element2-copilot-video-mode');
|
||
const videoTrackInput = document.getElementById('element2-copilot-video-track');
|
||
const senderModeInput = document.getElementById('element2-copilot-sender-mode');
|
||
const senderInput = document.getElementById('element2-copilot-sender');
|
||
audioModeInput.value = audioSelectionMode;
|
||
audioModeInput.addEventListener('change', () => {
|
||
writeAudioSelectionMode(audioModeInput.value);
|
||
if (audioSelectionMode === 'manual') {
|
||
manualAudioTrackId = selectedRemoteAudio?.id || selectableAudioItems()[0]?.track.id || null;
|
||
}
|
||
selectRemoteAudio('mode-change');
|
||
log(`AI input selection mode: ${audioSelectionMode}`);
|
||
});
|
||
audioTrackInput.addEventListener('change', () => {
|
||
manualAudioTrackId = audioTrackInput.value || null;
|
||
selectRemoteAudio('manual-select');
|
||
log(`manual AI input track: ${manualAudioTrackId || 'none'}`);
|
||
});
|
||
videoModeInput.value = videoSelectionMode;
|
||
videoModeInput.addEventListener('change', () => {
|
||
writeVideoSelectionMode(videoModeInput.value);
|
||
if (videoSelectionMode === 'manual') {
|
||
manualVideoTrackId = selectedRemoteVideo?.id || selectableVideoItems()[0]?.track.id || null;
|
||
}
|
||
videoTrackSelectSignature = '';
|
||
selectRemoteVideo('video-mode-change');
|
||
log(`AI video selection mode: ${videoSelectionMode}`);
|
||
});
|
||
videoTrackInput.addEventListener('change', () => {
|
||
manualVideoTrackId = videoTrackInput.value || null;
|
||
selectRemoteVideo('manual-video-select');
|
||
log(`manual AI video track: ${manualVideoTrackId || 'none'}`);
|
||
});
|
||
senderModeInput.value = senderSelectionMode;
|
||
senderModeInput.addEventListener('change', async () => {
|
||
writeSenderSelectionMode(senderModeInput.value);
|
||
if (senderSelectionMode === 'manual') {
|
||
const selectedItem = Array.from(businessSenders.values()).find((item) => item.sender === selectedBusinessSender);
|
||
manualSenderId = selectedItem?.id || selectableSenderItems()[0]?.id || null;
|
||
}
|
||
await changeBusinessSender('sender-mode-change');
|
||
log(`business sender selection mode: ${senderSelectionMode}`);
|
||
});
|
||
senderInput.addEventListener('change', async () => {
|
||
manualSenderId = senderInput.value || null;
|
||
await changeBusinessSender('manual-sender-select');
|
||
log(`manual business sender: ${manualSenderId || 'none'}`);
|
||
});
|
||
recordButton.addEventListener('click', toggleInputRecording);
|
||
senderRecordButton.addEventListener('click', toggleSenderRecording);
|
||
videoRecordButton.addEventListener('click', toggleVideoRecording);
|
||
toggle.addEventListener('click', async () => {
|
||
if (!isAiTakeover && !assistantInput.value.trim()) {
|
||
alert('请先填写助手 ID。');
|
||
return;
|
||
}
|
||
if (!isAiTakeover && (!usernameInput.value.trim() || !passwordInput.value)) {
|
||
alert('请填写管理员用户名和密码。');
|
||
return;
|
||
}
|
||
isAiTakeover = !isAiTakeover;
|
||
updateModeButton();
|
||
if (isAiTakeover) {
|
||
const connected = await connectAi(
|
||
urlInput.value.trim(),
|
||
assistantInput.value.trim(),
|
||
usernameInput.value.trim(),
|
||
passwordInput.value,
|
||
);
|
||
if (connected) {
|
||
await attachAiOutputToBusiness();
|
||
} else {
|
||
isAiTakeover = false;
|
||
updateModeButton();
|
||
}
|
||
} else {
|
||
await restoreBusinessAudio();
|
||
disconnectAi();
|
||
}
|
||
});
|
||
updateModeButton();
|
||
updateRecordButton();
|
||
updateSenderRecordButton();
|
||
updateVideoRecordButton();
|
||
selectRemoteAudio('panel-ready');
|
||
selectRemoteVideo('panel-ready');
|
||
selectBusinessSender('panel-ready');
|
||
updateGateLabel();
|
||
}
|
||
|
||
function renderPanelSoon() {
|
||
try { renderPanel(); } catch (error) { console.warn('[AI Element2 Gated] panel error', error); }
|
||
}
|
||
|
||
if (document.readyState === 'loading') {
|
||
document.addEventListener('DOMContentLoaded', renderPanelSoon, { once: true });
|
||
} else {
|
||
renderPanelSoon();
|
||
}
|
||
requestAnimationFrame(renderVu);
|
||
setInterval(refreshStats, 250);
|
||
setTimeout(renderPanelSoon, 1500);
|
||
setTimeout(renderPanelSoon, 3000);
|
||
window.addEventListener('beforeunload', () => {
|
||
if (inputRecorder?.state === 'recording') stopInputRecording();
|
||
if (senderRecorder?.state === 'recording') senderRecorder.stop();
|
||
if (videoRecorder?.state === 'recording') stopVideoRecording();
|
||
});
|
||
})();
|