Add support for Xfyun ASR and TTS services in the backend

- Introduce new Xfyun ASR and TTS services, enabling integration with iFlytek's voice recognition and synthesis capabilities.
- Update AssistantConfig model to include interface types for STT and TTS.
- Enhance credential testing to validate Xfyun credentials.
- Modify service factory to create Xfyun services based on configuration.
- Update README with new configuration details for Xfyun integration.
- Add new frontend components for visualizing audio streams and managing user interactions.
This commit is contained in:
Xin Wang
2026-06-11 10:51:08 +08:00
parent c69dec04e0
commit e25dfd4003
19 changed files with 1595 additions and 71 deletions

View File

@@ -74,6 +74,8 @@ export function useVoicePreview(assistantId: string | null) {
const [error, setError] = useState<string | null>(null);
const [micWarning, setMicWarning] = useState<string | null>(null);
const [localStream, setLocalStream] = useState<MediaStream | null>(null);
// 远端(助手 TTS)媒体流:除挂到 <audio> 播放外,也暴露给波形可视化
const [remoteStream, setRemoteStream] = useState<MediaStream | null>(null);
const [messages, setMessages] = useState<ChatMessage[]>([]);
// 可选麦克风列表与当前选择(空串表示交给浏览器选默认设备)
const [audioInputs, setAudioInputs] = useState<MediaDeviceInfo[]>([]);
@@ -92,7 +94,8 @@ export function useVoicePreview(assistantId: string | null) {
if (!navigator.mediaDevices?.enumerateDevices) return;
try {
const devices = await navigator.mediaDevices.enumerateDevices();
const inputs = devices.filter((d) => d.kind === "audioinput");
// 未授权时设备可能没有 deviceId(空串),无法选择,直接过滤掉
const inputs = devices.filter((d) => d.kind === "audioinput" && d.deviceId);
setAudioInputs(inputs);
// 选中的设备已被拔出时,回退到浏览器默认设备
if (
@@ -157,6 +160,7 @@ export function useVoicePreview(assistantId: string | null) {
const disconnect = useCallback(() => {
releaseResources();
setLocalStream(null);
setRemoteStream(null);
setError(null);
setMicWarning(null);
setStatus("idle");
@@ -166,6 +170,7 @@ export function useVoicePreview(assistantId: string | null) {
(message: string) => {
releaseResources();
setLocalStream(null);
setRemoteStream(null);
setError(message);
setStatus("failed");
},
@@ -312,9 +317,11 @@ export function useVoicePreview(assistantId: string | null) {
};
pc.ontrack = (e) => {
if (e.track.kind === "audio" && audioRef.current) {
audioRef.current.srcObject =
e.streams[0] ?? new MediaStream([e.track]);
if (e.track.kind !== "audio") return;
const remote = e.streams[0] ?? new MediaStream([e.track]);
setRemoteStream(remote);
if (audioRef.current) {
audioRef.current.srcObject = remote;
void audioRef.current.play().catch(() => {});
}
};
@@ -384,6 +391,7 @@ export function useVoicePreview(assistantId: string | null) {
error,
micWarning,
localStream,
remoteStream,
messages,
audioInputs,
selectedDeviceId,