diff --git a/scripts/tampermonkey/audio-track-probe.js b/scripts/tampermonkey/audio-track-probe.js index 0b33f13..620e758 100644 --- a/scripts/tampermonkey/audio-track-probe.js +++ b/scripts/tampermonkey/audio-track-probe.js @@ -473,7 +473,7 @@ 'top:20px', 'right:20px', 'width:460px', - 'max-height:70vh', + 'max-height:90vh', 'z-index:999999', 'box-sizing:border-box', 'padding:12px', @@ -494,7 +494,7 @@
Output senders
-
+
`; document.body.appendChild(panel); document.getElementById('audio-track-probe-copy').addEventListener('click', async () => { diff --git a/scripts/tampermonkey/copilot-element2-two-party.js b/scripts/tampermonkey/copilot-element2-two-party.js index 9d381df..a42545d 100644 --- a/scripts/tampermonkey/copilot-element2-two-party.js +++ b/scripts/tampermonkey/copilot-element2-two-party.js @@ -12,7 +12,7 @@ (function () { 'use strict'; - const DEFAULT_SIGNALING_URL = 'ws://182.92.86.220:8000/ws/voice'; + const DEFAULT_API_BASE_URL = 'http://182.92.86.220:8000'; const STATIC_AUDIO_ELEMENT_REASON = 'element-2:audio'; const STATIC_VIDEO_ELEMENT_REASON = 'element-0:video'; const PARTICIPANT_AUDIO_TRACK_PATTERN = /-audio-3-a$/; @@ -32,18 +32,39 @@ const internalPcs = new WeakSet(); const AUDIO_UPLINK_STORAGE_KEY = 'element2-copilot-send-audio'; const VIDEO_UPLINK_STORAGE_KEY = 'element2-copilot-send-video'; + const AUDIO_SELECTION_MODE_STORAGE_KEY = 'element2-copilot-audio-selection-mode'; + const VIDEO_SELECTION_MODE_STORAGE_KEY = 'element2-copilot-video-selection-mode'; + const SENDER_SELECTION_MODE_STORAGE_KEY = 'element2-copilot-sender-selection-mode'; let selectedRemoteAudio = null; + let audioSelectionMode = readAudioSelectionMode(); + let manualAudioTrackId = null; + let audioTrackSelectSignature = ''; let selectedRemoteVideo = null; + let videoSelectionMode = readVideoSelectionMode(); + let manualVideoTrackId = null; + let videoTrackSelectSignature = ''; let selectedBusinessSender = null; + let senderSelectionMode = readSenderSelectionMode(); + let manualSenderId = null; + let senderSelectSignature = ''; + let changingBusinessSender = false; let originalBusinessTrack = null; let aiAudioTrack = null; let aiPc = null; - let aiWs = null; let aiDc = null; let aiAudioSender = null; let aiVideoSender = null; + let aiPcId = null; + let aiOfferUrl = null; + let aiAssistantId = null; + let aiAuthorization = null; + let aiCandidateQueue = []; + let aiCandidateFlushTimer = null; + let aiKeepAliveTimer = null; + let aiNegotiating = false; + let aiCanSendCandidates = false; let isAiTakeover = false; let aiAttached = false; let aiOutputAudioContext = null; @@ -61,6 +82,10 @@ let inputRecordingChunks = []; let inputRecordingTrack = null; let inputRecordingStartedAt = 0; + let senderRecorder = null; + let senderRecordingChunks = []; + let senderRecordingTrack = null; + let senderRecordingStartedAt = 0; let videoRecorder = null; let videoRecordingChunks = []; let videoRecordingTrack = null; @@ -88,6 +113,55 @@ } } + function readAudioSelectionMode() { + try { + if (!shouldSendAudioToAi()) return 'off'; + return localStorage.getItem(AUDIO_SELECTION_MODE_STORAGE_KEY) === 'manual' ? 'manual' : 'auto'; + } catch { + return 'auto'; + } + } + + function writeAudioSelectionMode(mode) { + audioSelectionMode = ['auto', 'manual', 'off'].includes(mode) ? mode : 'auto'; + writeBoolSetting(AUDIO_UPLINK_STORAGE_KEY, audioSelectionMode !== 'off'); + try { + localStorage.setItem(AUDIO_SELECTION_MODE_STORAGE_KEY, audioSelectionMode); + } catch {} + } + + function readVideoSelectionMode() { + try { + if (!shouldSendVideoToAi()) return 'off'; + return localStorage.getItem(VIDEO_SELECTION_MODE_STORAGE_KEY) === 'manual' ? 'manual' : 'auto'; + } catch { + return 'auto'; + } + } + + function writeVideoSelectionMode(mode) { + videoSelectionMode = ['auto', 'manual', 'off'].includes(mode) ? mode : 'auto'; + writeBoolSetting(VIDEO_UPLINK_STORAGE_KEY, videoSelectionMode !== 'off'); + try { + localStorage.setItem(VIDEO_SELECTION_MODE_STORAGE_KEY, videoSelectionMode); + } catch {} + } + + function readSenderSelectionMode() { + try { + return localStorage.getItem(SENDER_SELECTION_MODE_STORAGE_KEY) === 'manual' ? 'manual' : 'auto'; + } catch { + return 'auto'; + } + } + + function writeSenderSelectionMode(mode) { + senderSelectionMode = mode === 'manual' ? 'manual' : 'auto'; + try { + localStorage.setItem(SENDER_SELECTION_MODE_STORAGE_KEY, senderSelectionMode); + } catch {} + } + function shouldSendAudioToAi() { return readBoolSetting(AUDIO_UPLINK_STORAGE_KEY, true); } @@ -152,6 +226,12 @@ return `ai-input-${trackPart}-${stamp}.webm`; } + function senderRecordingFileName(track) { + const stamp = new Date().toISOString().replace(/[:.]/g, '-'); + const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track'; + return `business-sender-${trackPart}-${stamp}.webm`; + } + function videoRecordingFileName(track) { const stamp = new Date().toISOString().replace(/[:.]/g, '-'); const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track'; @@ -221,15 +301,76 @@ const button = document.getElementById('element2-copilot-record'); if (!button) return; const recording = inputRecorder?.state === 'recording'; - button.textContent = recording ? '停止录制 AI 输入' : '录制 AI 输入'; + button.textContent = recording ? '■' : '●'; + button.title = recording ? '停止录制 AI 输入' : '录制 AI 输入'; + button.setAttribute('aria-label', button.title); button.style.background = recording ? '#dc2626' : '#2563eb'; } + async function startSenderRecording() { + if (senderRecorder?.state === 'recording') return; + selectBusinessSender('record-sender'); + const track = selectedBusinessSender?.track; + if (!isLive(track)) { + appendChatLine('录音', '当前没有可录制的业务 sender 音轨', true); + return; + } + if (!window.MediaRecorder) { + appendChatLine('录音', '当前浏览器不支持 MediaRecorder', true); + return; + } + senderRecordingChunks = []; + senderRecordingTrack = track; + senderRecordingStartedAt = Date.now(); + const mimeType = recorderMimeType(); + senderRecorder = new MediaRecorder(new MediaStream([track]), mimeType ? { mimeType } : undefined); + senderRecorder.ondataavailable = (event) => { + if (event.data?.size > 0) senderRecordingChunks.push(event.data); + }; + senderRecorder.onstop = () => { + const blob = new Blob(senderRecordingChunks, { type: senderRecorder?.mimeType || 'audio/webm' }); + const url = URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = url; + link.download = senderRecordingFileName(senderRecordingTrack); + document.body.appendChild(link); + link.click(); + link.remove(); + setTimeout(() => URL.revokeObjectURL(url), 30000); + const seconds = ((Date.now() - senderRecordingStartedAt) / 1000).toFixed(1); + appendChatLine('录音', `已保存 ${link.download}（${seconds}s，${(blob.size / 1024).toFixed(1)} KB）`, true); + senderRecorder = null; + senderRecordingChunks = []; + senderRecordingTrack = null; + updateSenderRecordButton(); + }; + senderRecorder.start(250); + appendChatLine('录音', `开始录制业务 sender：${trackLabel(track)}`, true); + updateSenderRecordButton(); + } + + function toggleSenderRecording() { + if (senderRecorder?.state === 'recording') senderRecorder.stop(); + else void startSenderRecording(); + } + + function updateSenderRecordButton() { + const button = document.getElementById('element2-copilot-record-sender'); + if (!button) return; + const recording = senderRecorder?.state === 'recording'; + button.textContent = recording ? '■' : '●'; + button.title = recording ? '停止录制 sender 音轨' : '录制 sender 音轨'; + button.setAttribute('aria-label', button.title); + button.style.background = recording ? '#dc2626' : '#7c3aed'; + } + function updateVideoRecordButton() { const button = document.getElementById('element2-copilot-record-video'); if (!button) return; const recording = videoRecorder?.state === 'recording'; - button.textContent = recording ? '停止录制 AI 视频' : '录制 AI 视频'; + button.textContent = recording ? '■' : '●'; + button.title = recording ? '停止录制 AI 视频' : '录制 AI 视频'; + button.setAttribute('aria-label', button.title); button.style.background = recording ? '#dc2626' : '#475569'; } @@ -334,10 +475,70 @@ return reasons[0] || 'audio-track'; } + function shortTrackId(track) { + if (!track?.id) return 'none'; + return track.id.length <= 22 ? track.id : `${track.id.slice(0, 10)}...${track.id.slice(-7)}`; + } + + function selectableAudioItems() { + return Array.from(remoteAudio.values()).filter((item) => ( + item?.receiver + && isLive(item.track) + && !aiOutputTrackIds.has(item.track.id) + )); + } + + function audioOptionLabel(item) { + const source = audioSourceLabel(item); + const state = item.track.muted ? '静音' : '可用'; + return `${source} · ${shortTrackId(item.track)} · ${state}`; + } + + function updateAudioTrackSelect() { + const select = document.getElementById('element2-copilot-audio-track'); + if (!select) return; + const items = selectableAudioItems().sort((a, b) => scoreAudioCandidate(b) - scoreAudioCandidate(a)); + const desiredValue = audioSelectionMode === 'manual' ? manualAudioTrackId : selectedRemoteAudio?.id; + const signature = [ + audioSelectionMode, + desiredValue || '', + ...items.map((item) => `${item.track.id}:${item.track.muted}:${audioSourceLabel(item)}`), + ].join('|'); + if (signature === audioTrackSelectSignature) return; + audioTrackSelectSignature = signature; + select.innerHTML = ''; + if (items.length === 0) { + const option = document.createElement('option'); + option.value = ''; + option.textContent = '暂无可用远端音轨'; + select.appendChild(option); + } else { + if (audioSelectionMode === 'manual' && !manualAudioTrackId) { + const placeholder = document.createElement('option'); + placeholder.value = ''; + placeholder.textContent = '请选择远端音轨'; + select.appendChild(placeholder); + } + for (const item of items) { + const option = document.createElement('option'); + option.value = item.track.id; + option.textContent = audioOptionLabel(item); + select.appendChild(option); + } + } + if (desiredValue && items.some((item) => item.track.id === desiredValue)) select.value = desiredValue; + select.disabled = audioSelectionMode !== 'manual'; + select.style.opacity = select.disabled ? '0.55' : '1'; + } + function updateSelectedAudioLabel() { const label = document.getElementById('element2-copilot-selected-audio'); if (!label) return; - label.textContent = '截取给 AI 的音频音量'; + const mode = audioSelectionMode === 'manual' ? '手动' : audioSelectionMode === 'off' ? '关闭' : '自动'; + const item = selectedRemoteAudio ? remoteAudio.get(selectedRemoteAudio.id) : null; + label.textContent = selectedRemoteAudio + ? `${mode}：${audioSourceLabel(item)} · ${shortTrackId(selectedRemoteAudio)}` + : audioSelectionMode === 'off' ? '不传音频给 AI' : `${mode}：尚未选中音轨`; } function pageHasParticipant() { @@ -350,29 +551,25 @@ return text.includes('坐席端'); } - function isParticipantAudioItem(candidate) { - return Boolean( - candidate - && isLive(candidate.track) - && !candidate.track.muted - && hasElementReason(candidate, STATIC_AUDIO_ELEMENT_REASON) - && PARTICIPANT_AUDIO_TRACK_PATTERN.test(candidate.track.id) - ); - } - - function staticAudioItem() { - return Array.from(remoteAudio.values()).find(isParticipantAudioItem) || null; + function selectedAudioItem() { + const item = selectedRemoteAudio ? remoteAudio.get(selectedRemoteAudio.id) : null; + return item?.receiver && isLive(item.track) && !item.track.muted ? item : null; } function gateStatus() { - const item = staticAudioItem(); + const item = selectedAudioItem(); + const participantAudioPresent = selectableAudioItems().some((candidate) => !candidate.track.muted); + const senderItem = Array.from(businessSenders.values()).find((candidate) => ( + candidate.sender === selectedBusinessSender + && selectableSenderItems().includes(candidate) + )); const participantVisible = pageHasParticipant(); const operatorVisible = pageHasOperator(); return { bothParties: participantVisible && operatorVisible, - participantOnline: participantVisible && Boolean(item), + participantOnline: participantVisible && (Boolean(item) || (audioSelectionMode === 'off' && participantAudioPresent)), operatorOnline: operatorVisible, - senderReady: Boolean(selectedBusinessSender), + senderReady: Boolean(senderItem), track: item?.track || null, }; } @@ -401,6 +598,8 @@ if ((item.packetsDelta || 0) > 0) score += 2000 + Math.min(800, item.packetsDelta * 8); score += Math.min(1200, (item.audioLevel || 0) * 12000); if (elementAudioReasons(item).length > 0) score += 100; + if (hasElementReason(item, STATIC_AUDIO_ELEMENT_REASON)) score += 5000; + if (PARTICIPANT_AUDIO_TRACK_PATTERN.test(item.track.id)) score += 5000; item.audioScore = score; return score; } @@ -464,50 +663,122 @@ } function selectRemoteAudio(reason) { + const items = selectableAudioItems(); const current = selectedRemoteAudio && remoteAudio.get(selectedRemoteAudio.id); - if (isParticipantAudioItem(current)) { - updateGateLabel(); - return selectedRemoteAudio; + let item = null; + if (audioSelectionMode === 'off') { + item = null; + } else if (audioSelectionMode === 'manual') { + item = items.find((candidate) => candidate.track.id === manualAudioTrackId) || null; + } else { + const ranked = items + .filter((candidate) => !candidate.track.muted) + .sort((a, b) => scoreAudioCandidate(b) - scoreAudioCandidate(a)); + const best = ranked[0] || null; + const currentScore = current ? scoreAudioCandidate(current) : -1; + const bestScore = best ? scoreAudioCandidate(best) : -1; + item = current && currentScore >= 0 && (!best || bestScore < currentScore + 1500) + ? current + : best; } - - const item = staticAudioItem(); if (!item) { + if (selectedRemoteAudio) { + selectedRemoteAudio = null; + if (aiAudioSender) void aiAudioSender.replaceTrack(null).catch(() => {}); + } updateSelectedAudioLabel(); + updateAudioTrackSelect(); updateGateLabel(); return null; } if (selectedRemoteAudio !== item.track) { selectedRemoteAudio = item.track; - log(`selected input audio (${reason}) from ${STATIC_AUDIO_ELEMENT_REASON}: ${trackLabel(selectedRemoteAudio)}`, true); + log(`selected input audio (${reason}) from ${audioSourceLabel(item)}: ${trackLabel(selectedRemoteAudio)}`, true); updateSelectedAudioLabel(); attachAudioToAi(); } + updateAudioTrackSelect(); updateGateLabel(); return selectedRemoteAudio; } function selectRemoteVideo(reason) { - const current = selectedRemoteVideo && remoteVideo.get(selectedRemoteVideo.id); - if (isLive(selectedRemoteVideo) && !selectedRemoteVideo.muted && hasElementReason(current, STATIC_VIDEO_ELEMENT_REASON)) { - return selectedRemoteVideo; + const items = selectableVideoItems(); + let item = null; + if (videoSelectionMode === 'manual') { + item = items.find((candidate) => candidate.track.id === manualVideoTrackId) || null; + } else if (videoSelectionMode === 'auto') { + item = items.find((candidate) => hasElementReason(candidate, STATIC_VIDEO_ELEMENT_REASON)) || items[0] || null; + } + if (!item) { + if (selectedRemoteVideo) { + selectedRemoteVideo = null; + if (aiVideoSender) void aiVideoSender.replaceTrack(null).catch(() => {}); + } + updateVideoTrackSelect(); + return null; } - - const item = Array.from(remoteVideo.values()).find((candidate) => ( - isLive(candidate.track) - && !candidate.track.muted - && hasElementReason(candidate, STATIC_VIDEO_ELEMENT_REASON) - )); - if (!item) return null; if (selectedRemoteVideo !== item.track) { selectedRemoteVideo = item.track; - log(`selected input video (${reason}) from ${STATIC_VIDEO_ELEMENT_REASON}: ${trackLabel(selectedRemoteVideo)}`, true); + log(`selected input video (${reason}): ${trackLabel(selectedRemoteVideo)}`, true); attachVideoToAi(); } + updateVideoTrackSelect(); return selectedRemoteVideo; } + function selectableVideoItems() { + return Array.from(remoteVideo.values()).filter((item) => ( + item?.receiver + && isLive(item.track) + && !item.track.muted + )); + } + + function videoSourceLabel(item) { + const reason = Array.from(item?.reasons || []).find((value) => /^element-\d+:video$/.test(value)); + return reason || 'video-track'; + } + + function updateVideoTrackSelect() { + const select = document.getElementById('element2-copilot-video-track'); + if (!select) return; + const items = selectableVideoItems(); + const desiredValue = videoSelectionMode === 'manual' ? manualVideoTrackId : selectedRemoteVideo?.id; + const signature = [ + videoSelectionMode, + desiredValue || '', + ...items.map((item) => `${item.track.id}:${videoSourceLabel(item)}`), + ].join('|'); + if (signature === videoTrackSelectSignature) return; + videoTrackSelectSignature = signature; + select.innerHTML = ''; + if (items.length === 0) { + const option = document.createElement('option'); + option.value = ''; + option.textContent = '暂无可用远端视频'; + select.appendChild(option); + } else { + if (videoSelectionMode === 'manual' && !manualVideoTrackId) { + const placeholder = document.createElement('option'); + placeholder.value = ''; + placeholder.textContent = '请选择远端视频'; + select.appendChild(placeholder); + } + for (const item of items) { + const option = document.createElement('option'); + option.value = item.track.id; + option.textContent = `${videoSourceLabel(item)} · ${shortTrackId(item.track)}`; + select.appendChild(option); + } + } + if (desiredValue && items.some((item) => item.track.id === desiredValue)) select.value = desiredValue; + select.disabled = videoSelectionMode !== 'manual'; + select.style.opacity = select.disabled ? '0.55' : '1'; + } + function rememberBusinessSender(sender, track, reason) { if (!sender) return; const senderTrack = track || sender.track; @@ -544,26 +815,121 @@ } function isRemoteInputTrack(track) { - return Boolean(track && remoteAudio.has(track.id) && !aiOutputTrackIds.has(track.id)); + const item = track ? remoteAudio.get(track.id) : null; + return Boolean(item?.receiver && !aiOutputTrackIds.has(track.id)); + } + + function senderCandidateTrack(item) { + const current = item?.sender?.track; + if (current && current !== aiAudioTrack) return current; + return item?.originalTrack || item?.track || null; + } + + function selectableSenderItems() { + return Array.from(businessSenders.values()).filter((item) => { + const track = senderCandidateTrack(item); + return isLive(track) && track.kind === 'audio' && !isRemoteInputTrack(track); + }); + } + + function scoreSenderCandidate(item) { + const track = senderCandidateTrack(item); + if (!isLive(track) || isRemoteInputTrack(track)) return -1; + let score = 1000; + if (!track.muted) score += 400; + if (item.sources.has('pc-addTrack')) score += 2000; + if ((item.bytesSent || 0) > 0) score += 300; + if ((item.bytesDelta || 0) > 0) score += 4500 + Math.min(1800, item.bytesDelta); + if ((item.packetsDelta || 0) > 0) score += 1800 + Math.min(900, item.packetsDelta * 10); + return score; + } + + function updateSenderSelect() { + const select = document.getElementById('element2-copilot-sender'); + if (!select) return; + const items = selectableSenderItems().sort((a, b) => scoreSenderCandidate(b) - scoreSenderCandidate(a)); + const selectedItem = items.find((item) => item.sender === selectedBusinessSender); + const desiredValue = senderSelectionMode === 'manual' ? manualSenderId : selectedItem?.id; + const signature = [ + senderSelectionMode, + desiredValue || '', + ...items.map((item) => `${item.id}:${senderCandidateTrack(item)?.id || ''}:${senderCandidateTrack(item)?.muted}`), + ].join('|'); + if (signature === senderSelectSignature) return; + senderSelectSignature = signature; + select.innerHTML = ''; + if (items.length === 0) { + const option = document.createElement('option'); + option.value = ''; + option.textContent = '暂无可用 sender'; + select.appendChild(option); + } else { + if (senderSelectionMode === 'manual' && !manualSenderId) { + const placeholder = document.createElement('option'); + placeholder.value = ''; + placeholder.textContent = '请选择 sender'; + select.appendChild(placeholder); + } + for (const item of items) { + const option = document.createElement('option'); + option.value = item.id; + option.textContent = `${item.id} · ${shortTrackId(senderCandidateTrack(item))}${item.sources.has('pc-addTrack') ? ' · addTrack' : ''}`; + select.appendChild(option); + } + } + if (desiredValue && items.some((item) => item.id === desiredValue)) select.value = desiredValue; + select.disabled = senderSelectionMode !== 'manual'; + select.style.opacity = select.disabled ? '0.55' : '1'; } function selectBusinessSender(reason) { - if (selectedBusinessSender && !isRemoteInputTrack(selectedBusinessSender.track)) return selectedBusinessSender; - - const item = Array.from(businessSenders.values()).find((candidate) => { - const track = candidate.sender.track || candidate.track || candidate.originalTrack; - return candidate.sources.has('pc-addTrack') && !isRemoteInputTrack(track); - }); - if (!item) return null; + if (changingBusinessSender) return selectedBusinessSender; + const items = selectableSenderItems(); + const current = items.find((candidate) => candidate.sender === selectedBusinessSender); + let item = null; + if (senderSelectionMode === 'manual') { + item = items.find((candidate) => candidate.id === manualSenderId) || null; + } else { + item = current || items.sort((a, b) => scoreSenderCandidate(b) - scoreSenderCandidate(a))[0] || null; + } + if (!item) { + if (!aiAttached) { + selectedBusinessSender = null; + originalBusinessTrack = null; + } + updateSenderSelect(); + return null; + } + if (selectedBusinessSender === item.sender) { + updateSenderSelect(); + return selectedBusinessSender; + } selectedBusinessSender = item.sender; originalBusinessTrack = item.originalTrack || item.sender.track || originalBusinessTrack; log(`selected output sender (${reason}): ${item.id}, track=${trackLabel(item.sender.track)}`, true); + updateSenderSelect(); updateGateLabel(); attachAiOutputToBusiness(); return selectedBusinessSender; } + async function changeBusinessSender(reason) { + changingBusinessSender = true; + try { + await restoreBusinessAudio(); + selectedBusinessSender = null; + originalBusinessTrack = null; + aiAttached = false; + } finally { + changingBusinessSender = false; + } + senderSelectSignature = ''; + selectBusinessSender(reason); + if (isAiTakeover) await attachAiOutputToBusiness(); + updateGateLabel(); + } + function setVu(id, level) { const el = document.getElementById(id); if (!el) return; @@ -740,6 +1106,8 @@ outboundMemory.set(key, { bytes, packets }); }); } + selectBusinessSender('stats'); + updateSenderSelect(); } async function outboundBytes(sender) { @@ -949,7 +1317,133 @@ setInterval(scanMediaElements, 1000); - async function connectAi(signalingUrl, assistantId) { + function endpointUrl(baseUrl, path) { + const normalizedBaseUrl = baseUrl.trim().replace(/\/+$/, ''); + return new URL(`${normalizedBaseUrl}${path}`).toString(); + } + + function basicAuthorization(username, password) { + const bytes = new TextEncoder().encode(`${username}:${password}`); + let binary = ''; + bytes.forEach((byte) => { binary += String.fromCharCode(byte); }); + return `Basic ${btoa(binary)}`; + } + + function aiRequestHeaders(extra = {}) { + return { + ...extra, + ...(aiAuthorization ? { Authorization: aiAuthorization } : {}), + }; + } + + async function fetchIceServers(baseUrl) { + try { + const iceUrl = endpointUrl(baseUrl, '/api/webrtc/ice-servers'); + const response = await fetch(iceUrl, { + credentials: 'include', + headers: aiRequestHeaders(), + }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const payload = await response.json(); + if (Array.isArray(payload.iceServers)) return payload.iceServers; + } catch (error) { + log(`ICE server config unavailable, using public STUN: ${error.message || error}`); + } + return [{ urls: 'stun:stun.l.google.com:19302' }]; + } + + function queueAiIceCandidate(candidate) { + if (!candidate) return; + aiCandidateQueue.push({ + candidate: candidate.candidate, + sdp_mid: candidate.sdpMid, + sdp_mline_index: candidate.sdpMLineIndex, + }); + if (!aiCandidateFlushTimer) { + aiCandidateFlushTimer = setTimeout(flushAiIceCandidates, 200); + } + } + + async function flushAiIceCandidates() { + aiCandidateFlushTimer = null; + if (!aiOfferUrl || !aiPcId || !aiCanSendCandidates || aiCandidateQueue.length === 0) return; + const candidates = aiCandidateQueue.splice(0, aiCandidateQueue.length); + try { + const response = await fetch(aiOfferUrl, { + method: 'PATCH', + credentials: 'include', + headers: aiRequestHeaders({ 'Content-Type': 'application/json' }), + body: JSON.stringify({ pc_id: aiPcId, candidates }), + }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + } catch (error) { + aiCandidateQueue.unshift(...candidates); + log(`failed to send ICE candidates: ${error.message || error}`); + } + } + + function sendAiSignalling(message) { + if (aiDc?.readyState !== 'open') return; + aiDc.send(JSON.stringify({ type: 'signalling', message })); + } + + async function negotiateAi(restartPc = false) { + if (!aiPc || !aiOfferUrl || !aiAssistantId || aiNegotiating) return false; + aiNegotiating = true; + try { + const offer = await aiPc.createOffer(); + await aiPc.setLocalDescription(offer); + const response = await fetch(aiOfferUrl, { + method: 'POST', + credentials: 'include', + headers: aiRequestHeaders({ + 'Content-Type': 'application/json', + 'X-Pipecat-Assistant-ID': aiAssistantId, + }), + body: JSON.stringify({ + pc_id: aiPcId, + sdp: aiPc.localDescription.sdp, + type: aiPc.localDescription.type, + restart_pc: restartPc, + requestData: { + assistant_id: aiAssistantId, + vision_enabled: shouldSendVideoToAi(), + dynamic_variables: {}, + }, + }), + }); + if (!response.ok) { + const detail = await response.text().catch(() => ''); + throw new Error(`HTTP ${response.status}${detail ? `: ${detail}` : ''}`); + } + const answer = await response.json(); + aiPcId = answer.pc_id; + await aiPc.setRemoteDescription({ type: answer.type, sdp: answer.sdp }); + aiCanSendCandidates = true; + await flushAiIceCandidates(); + log(`AI WebRTC answer applied; ${aiState()}`, true); + return true; + } finally { + aiNegotiating = false; + } + } + + async function handleAiSignalling(message) { + try { + if (message?.type === 'renegotiate') { + await negotiateAi(false); + } else if (message?.type === 'peerLeft') { + setStatus('disconnected'); + log('AI peer left', true); + } else { + log(`unknown SmallWebRTC signalling: ${message?.type || 'missing type'}`); + } + } catch (error) { + log(`SmallWebRTC renegotiation failed: ${error.message || error}`, true); + } + } + + async function connectAi(baseUrl, assistantId, username, password) { scanMediaElements(); selectRemoteAudio('connect'); selectRemoteVideo('connect'); @@ -965,8 +1459,20 @@ disconnectAi(); selectedRemoteAudio = status.track; - const id = pcId(); - aiPc = new RTCPeerConnection({ iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] }); + try { + aiOfferUrl = endpointUrl(baseUrl, '/api/webrtc/offer'); + } catch { + appendChatLine('配置', 'Base URL 格式无效，请填写包含 http:// 或 https:// 的完整地址', true); + setStatus('failed'); + return false; + } + aiAssistantId = assistantId; + aiAuthorization = basicAuthorization(username, password); + aiPcId = pcId(); + aiCandidateQueue = []; + aiCanSendCandidates = false; + const iceServers = await fetchIceServers(baseUrl); + aiPc = new RTCPeerConnection({ iceServers }); markInternalPc(aiPc); aiPc.onconnectionstatechange = () => log(`AI PC connectionState=${aiPc.connectionState}`); aiPc.oniceconnectionstatechange = () => log(`AI PC iceConnectionState=${aiPc.iceConnectionState}`); @@ -975,14 +1481,21 @@ aiDc.onopen = () => { setStatus('connected'); log('AI data channel open', true); + sendAiSignalling({ type: 'trackStatus', receiver_index: 0, enabled: shouldSendAudioToAi() }); + sendAiSignalling({ type: 'trackStatus', receiver_index: 1, enabled: shouldSendVideoToAi() }); aiDc.send(JSON.stringify({ type: 'client-ready' })); + aiKeepAliveTimer = setInterval(() => { + if (aiDc?.readyState === 'open') aiDc.send(`ping: ${Date.now()}`); + }, 1000); }; aiDc.onclose = () => setStatus('disconnected'); aiDc.onmessage = (event) => { try { const msg = JSON.parse(event.data); const content = msg.delta || msg.content; - if (msg.type === 'transcript') { + if (msg.type === 'signalling') { + void handleAiSignalling(msg.message); + } else if (msg.type === 'transcript') { logUserTranscript(content); } else if (msg.type === 'assistant-text-start') { startAssistantLine(); @@ -999,10 +1512,9 @@ }; const sendAudio = shouldSendAudioToAi(); - const audioTransceiver = sendAudio && isLive(selectedRemoteAudio) - ? aiPc.addTransceiver(selectedRemoteAudio, { direction: 'sendrecv' }) - : aiPc.addTransceiver('audio', { direction: sendAudio ? 'sendrecv' : 'recvonly' }); + const audioTransceiver = aiPc.addTransceiver('audio', { direction: 'sendrecv' }); aiAudioSender = audioTransceiver.sender; + if (sendAudio && isLive(selectedRemoteAudio)) await aiAudioSender.replaceTrack(selectedRemoteAudio); log(!sendAudio ? 'AI audio uplink disabled by panel setting' : isLive(selectedRemoteAudio) @@ -1011,12 +1523,12 @@ const sendVideo = shouldSendVideoToAi(); const shouldSendVideo = sendVideo && isLive(selectedRemoteVideo) && !selectedRemoteVideo.muted; + const videoTransceiver = aiPc.addTransceiver('video', { direction: 'sendrecv' }); + aiVideoSender = videoTransceiver.sender; if (shouldSendVideo) { - const videoTransceiver = aiPc.addTransceiver(selectedRemoteVideo, { direction: 'sendonly' }); - aiVideoSender = videoTransceiver.sender; + await aiVideoSender.replaceTrack(selectedRemoteVideo); log(`AI video starts with ${trackLabel(selectedRemoteVideo)} (${STATIC_VIDEO_ELEMENT_REASON})`); } else { - aiVideoSender = null; log(sendVideo ? `AI video disabled: waiting for live track at ${STATIC_VIDEO_ELEMENT_REASON}` : 'AI video uplink disabled by panel setting'); @@ -1037,57 +1549,19 @@ }; aiPc.onicecandidate = (event) => { - if (aiWs?.readyState !== WebSocket.OPEN) return; - aiWs.send(JSON.stringify({ - type: 'ice-candidate', - payload: { - pc_id: id, - candidate: event.candidate ? { - candidate: event.candidate.candidate, - sdpMid: event.candidate.sdpMid, - sdpMLineIndex: event.candidate.sdpMLineIndex, - } : null, - }, - })); + queueAiIceCandidate(event.candidate); }; setStatus('connecting'); - aiWs = new WebSocket(signalingUrl); - aiWs.onopen = async () => { - const offer = await aiPc.createOffer(); - await aiPc.setLocalDescription(offer); - aiWs.send(JSON.stringify({ - type: 'offer', - payload: { - pc_id: id, - sdp: aiPc.localDescription.sdp, - type: aiPc.localDescription.type, - assistant_id: assistantId, - vision_enabled: shouldSendVideo, - }, - })); - log('sent AI WebRTC offer', true); - }; - aiWs.onmessage = async (event) => { - try { - const msg = JSON.parse(event.data); - if (msg.type === 'answer') { - await aiPc.setRemoteDescription({ type: 'answer', sdp: msg.payload.sdp }); - log(`AI WebRTC answer applied; ${aiState()}`, true); - } else if (msg.type === 'ice-candidate' && msg.payload?.candidate) { - await aiPc.addIceCandidate(msg.payload.candidate).catch(() => {}); - } else if (msg.type === 'error') { - throw new Error(msg.payload?.message || 'server error'); - } - } catch (error) { - log(`signaling error: ${error.message || error}`); - } - }; - aiWs.onerror = () => { + try { + await negotiateAi(false); + log('sent SmallWebRTC HTTP offer', true); + } catch (error) { + log(`AI HTTP signalling error: ${error.message || error}`, true); + disconnectAi(); setStatus('failed'); - log('AI signaling websocket error'); - }; - aiWs.onclose = () => setStatus('disconnected'); + return false; + } attachAudioToAi(); attachVideoToAi(); @@ -1097,15 +1571,25 @@ function disconnectAi() { try { aiDc?.close(); } catch {} try { aiPc?.close(); } catch {} - try { aiWs?.close(); } catch {} + if (aiCandidateFlushTimer) clearTimeout(aiCandidateFlushTimer); + if (aiKeepAliveTimer) clearInterval(aiKeepAliveTimer); aiDc = null; aiPc = null; - aiWs = null; + aiPcId = null; + aiOfferUrl = null; + aiAssistantId = null; + aiAuthorization = null; + aiCandidateQueue = []; + aiCandidateFlushTimer = null; + aiKeepAliveTimer = null; + aiNegotiating = false; + aiCanSendCandidates = false; aiAudioSender = null; aiVideoSender = null; aiAudioTrack = null; detachAiOutputMeter(); if (inputRecorder?.state === 'recording') stopInputRecording(); + if (senderRecorder?.state === 'recording') senderRecorder.stop(); if (videoRecorder?.state === 'recording') stopVideoRecording(); setStatus('disconnected'); } @@ -1136,7 +1620,20 @@ window.__aiElement2GatedDump = function () { const status = gateStatus(); return { - audioSelection: STATIC_AUDIO_ELEMENT_REASON, + audioSelection: { + mode: audioSelectionMode, + manualTrackId: manualAudioTrackId, + preferredElement: STATIC_AUDIO_ELEMENT_REASON, + }, + videoSelection: { + mode: videoSelectionMode, + manualTrackId: manualVideoTrackId, + preferredElement: STATIC_VIDEO_ELEMENT_REASON, + }, + senderSelection: { + mode: senderSelectionMode, + manualSenderId, + }, staticVideoElement: STATIC_VIDEO_ELEMENT_REASON, gate: { bothParties: status.bothParties, @@ -1159,6 +1656,11 @@ seconds: ((Date.now() - inputRecordingStartedAt) / 1000).toFixed(1), chunks: inputRecordingChunks.length, } : null, + senderRecording: senderRecorder?.state === 'recording' ? { + track: trackLabel(senderRecordingTrack), + seconds: ((Date.now() - senderRecordingStartedAt) / 1000).toFixed(1), + chunks: senderRecordingChunks.length, + } : null, videoRecording: videoRecorder?.state === 'recording' ? { track: trackLabel(videoRecordingTrack), seconds: ((Date.now() - videoRecordingStartedAt) / 1000).toFixed(1), @@ -1213,7 +1715,9 @@ 'position:fixed', 'top:20px', 'left:20px', - 'width:360px', + 'width:440px', + 'max-height:calc(100vh - 40px)', + 'overflow:auto', 'z-index:999999', 'box-sizing:border-box', 'padding:16px', @@ -1231,33 +1735,87 @@
等待当事人在线
- - - - -
- - +
+
+ + +
+
+ + +
-
+
+
+ + +
+
+ + +
+
+
+
+
+ + +
+
+ + +
+ +
截取给 AI 的音频音量vol=0% -inf
-
+
+
+
+ + +
+
+ + +
+ +
AI 发送给当事人的音频音量vol=0% -inf
- -
- - +
+
+
+ + +
+
+ + +
+ +
+
等待交互...
@@ -1266,32 +1824,83 @@ const toggle = document.getElementById('element2-copilot-toggle'); const recordButton = document.getElementById('element2-copilot-record'); + const senderRecordButton = document.getElementById('element2-copilot-record-sender'); const videoRecordButton = document.getElementById('element2-copilot-record-video'); const urlInput = document.getElementById('element2-copilot-url'); const assistantInput = document.getElementById('element2-copilot-assistant'); - const sendAudioInput = document.getElementById('element2-copilot-send-audio'); - const sendVideoInput = document.getElementById('element2-copilot-send-video'); - sendAudioInput.checked = shouldSendAudioToAi(); - sendVideoInput.checked = shouldSendVideoToAi(); - sendAudioInput.addEventListener('change', () => { - writeBoolSetting(AUDIO_UPLINK_STORAGE_KEY, sendAudioInput.checked); - log(`AI audio uplink setting: ${sendAudioInput.checked ? 'on' : 'off'}`); + const usernameInput = document.getElementById('element2-copilot-username'); + const passwordInput = document.getElementById('element2-copilot-password'); + const audioModeInput = document.getElementById('element2-copilot-audio-mode'); + const audioTrackInput = document.getElementById('element2-copilot-audio-track'); + const videoModeInput = document.getElementById('element2-copilot-video-mode'); + const videoTrackInput = document.getElementById('element2-copilot-video-track'); + const senderModeInput = document.getElementById('element2-copilot-sender-mode'); + const senderInput = document.getElementById('element2-copilot-sender'); + audioModeInput.value = audioSelectionMode; + audioModeInput.addEventListener('change', () => { + writeAudioSelectionMode(audioModeInput.value); + if (audioSelectionMode === 'manual') { + manualAudioTrackId = selectedRemoteAudio?.id || selectableAudioItems()[0]?.track.id || null; + } + selectRemoteAudio('mode-change'); + log(`AI input selection mode: ${audioSelectionMode}`); }); - sendVideoInput.addEventListener('change', () => { - writeBoolSetting(VIDEO_UPLINK_STORAGE_KEY, sendVideoInput.checked); - log(`AI video uplink setting: ${sendVideoInput.checked ? 'on' : 'off'}`); + audioTrackInput.addEventListener('change', () => { + manualAudioTrackId = audioTrackInput.value || null; + selectRemoteAudio('manual-select'); + log(`manual AI input track: ${manualAudioTrackId || 'none'}`); + }); + videoModeInput.value = videoSelectionMode; + videoModeInput.addEventListener('change', () => { + writeVideoSelectionMode(videoModeInput.value); + if (videoSelectionMode === 'manual') { + manualVideoTrackId = selectedRemoteVideo?.id || selectableVideoItems()[0]?.track.id || null; + } + videoTrackSelectSignature = ''; + selectRemoteVideo('video-mode-change'); + log(`AI video selection mode: ${videoSelectionMode}`); + }); + videoTrackInput.addEventListener('change', () => { + manualVideoTrackId = videoTrackInput.value || null; + selectRemoteVideo('manual-video-select'); + log(`manual AI video track: ${manualVideoTrackId || 'none'}`); + }); + senderModeInput.value = senderSelectionMode; + senderModeInput.addEventListener('change', async () => { + writeSenderSelectionMode(senderModeInput.value); + if (senderSelectionMode === 'manual') { + const selectedItem = Array.from(businessSenders.values()).find((item) => item.sender === selectedBusinessSender); + manualSenderId = selectedItem?.id || selectableSenderItems()[0]?.id || null; + } + await changeBusinessSender('sender-mode-change'); + log(`business sender selection mode: ${senderSelectionMode}`); + }); + senderInput.addEventListener('change', async () => { + manualSenderId = senderInput.value || null; + await changeBusinessSender('manual-sender-select'); + log(`manual business sender: ${manualSenderId || 'none'}`); }); recordButton.addEventListener('click', toggleInputRecording); + senderRecordButton.addEventListener('click', toggleSenderRecording); videoRecordButton.addEventListener('click', toggleVideoRecording); toggle.addEventListener('click', async () => { if (!isAiTakeover && !assistantInput.value.trim()) { alert('请先填写助手 ID。'); return; } + if (!isAiTakeover && (!usernameInput.value.trim() || !passwordInput.value)) { + alert('请填写管理员用户名和密码。'); + return; + } isAiTakeover = !isAiTakeover; updateModeButton(); if (isAiTakeover) { - const connected = await connectAi(urlInput.value.trim(), assistantInput.value.trim()); + const connected = await connectAi( + urlInput.value.trim(), + assistantInput.value.trim(), + usernameInput.value.trim(), + passwordInput.value, + ); if (connected) { await attachAiOutputToBusiness(); } else { @@ -1305,7 +1914,11 @@ }); updateModeButton(); updateRecordButton(); + updateSenderRecordButton(); updateVideoRecordButton(); + selectRemoteAudio('panel-ready'); + selectRemoteVideo('panel-ready'); + selectBusinessSender('panel-ready'); updateGateLabel(); } @@ -1324,6 +1937,7 @@ setTimeout(renderPanelSoon, 3000); window.addEventListener('beforeunload', () => { if (inputRecorder?.state === 'recording') stopInputRecording(); + if (senderRecorder?.state === 'recording') senderRecorder.stop(); if (videoRecorder?.state === 'recording') stopVideoRecording(); }); })();