Add support for Xfyun ASR and TTS services in the backend
- Introduce new Xfyun ASR and TTS services, enabling integration with iFlytek's voice recognition and synthesis capabilities. - Update AssistantConfig model to include interface types for STT and TTS. - Enhance credential testing to validate Xfyun credentials. - Modify service factory to create Xfyun services based on configuration. - Update README with new configuration details for Xfyun integration. - Add new frontend components for visualizing audio streams and managing user interactions.
This commit is contained in:
@@ -74,6 +74,8 @@ export function useVoicePreview(assistantId: string | null) {
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [micWarning, setMicWarning] = useState<string | null>(null);
|
||||
const [localStream, setLocalStream] = useState<MediaStream | null>(null);
|
||||
// 远端(助手 TTS)媒体流:除挂到 <audio> 播放外,也暴露给波形可视化
|
||||
const [remoteStream, setRemoteStream] = useState<MediaStream | null>(null);
|
||||
const [messages, setMessages] = useState<ChatMessage[]>([]);
|
||||
// 可选麦克风列表与当前选择(空串表示交给浏览器选默认设备)
|
||||
const [audioInputs, setAudioInputs] = useState<MediaDeviceInfo[]>([]);
|
||||
@@ -92,7 +94,8 @@ export function useVoicePreview(assistantId: string | null) {
|
||||
if (!navigator.mediaDevices?.enumerateDevices) return;
|
||||
try {
|
||||
const devices = await navigator.mediaDevices.enumerateDevices();
|
||||
const inputs = devices.filter((d) => d.kind === "audioinput");
|
||||
// 未授权时设备可能没有 deviceId(空串),无法选择,直接过滤掉
|
||||
const inputs = devices.filter((d) => d.kind === "audioinput" && d.deviceId);
|
||||
setAudioInputs(inputs);
|
||||
// 选中的设备已被拔出时,回退到浏览器默认设备
|
||||
if (
|
||||
@@ -157,6 +160,7 @@ export function useVoicePreview(assistantId: string | null) {
|
||||
const disconnect = useCallback(() => {
|
||||
releaseResources();
|
||||
setLocalStream(null);
|
||||
setRemoteStream(null);
|
||||
setError(null);
|
||||
setMicWarning(null);
|
||||
setStatus("idle");
|
||||
@@ -166,6 +170,7 @@ export function useVoicePreview(assistantId: string | null) {
|
||||
(message: string) => {
|
||||
releaseResources();
|
||||
setLocalStream(null);
|
||||
setRemoteStream(null);
|
||||
setError(message);
|
||||
setStatus("failed");
|
||||
},
|
||||
@@ -312,9 +317,11 @@ export function useVoicePreview(assistantId: string | null) {
|
||||
};
|
||||
|
||||
pc.ontrack = (e) => {
|
||||
if (e.track.kind === "audio" && audioRef.current) {
|
||||
audioRef.current.srcObject =
|
||||
e.streams[0] ?? new MediaStream([e.track]);
|
||||
if (e.track.kind !== "audio") return;
|
||||
const remote = e.streams[0] ?? new MediaStream([e.track]);
|
||||
setRemoteStream(remote);
|
||||
if (audioRef.current) {
|
||||
audioRef.current.srcObject = remote;
|
||||
void audioRef.current.play().catch(() => {});
|
||||
}
|
||||
};
|
||||
@@ -384,6 +391,7 @@ export function useVoicePreview(assistantId: string | null) {
|
||||
error,
|
||||
micWarning,
|
||||
localStream,
|
||||
remoteStream,
|
||||
messages,
|
||||
audioInputs,
|
||||
selectedDeviceId,
|
||||
|
||||
Reference in New Issue
Block a user