Voice library support dashscope

api/app/routers/assistants.py

@@ -34,6 +34,12 @@ def _is_openai_compatible_vendor(vendor: Optional[str]) -> bool:
     }
 
 
+def _is_dashscope_vendor(vendor: Optional[str]) -> bool:
+    return (vendor or "").strip().lower() in {
+        "dashscope",
+    }
+
+
 def _normalize_openai_compatible_voice_key(voice_value: str, model: str) -> str:
     raw = (voice_value or "").strip()
     model_name = (model or "").strip() or OPENAI_COMPATIBLE_DEFAULT_MODEL
@@ -121,7 +127,12 @@ def _resolve_runtime_metadata(db: Session, assistant: Assistant) -> tuple[Dict[s
     elif assistant.voice:
         voice = db.query(Voice).filter(Voice.id == assistant.voice).first()
         if voice:
-            tts_provider = "openai_compatible" if _is_openai_compatible_vendor(voice.vendor) else "edge"
+            if _is_dashscope_vendor(voice.vendor):
+                tts_provider = "dashscope"
+            elif _is_openai_compatible_vendor(voice.vendor):
+                tts_provider = "openai_compatible"
+            else:
+                tts_provider = "edge"
             model = voice.model
             runtime_voice = voice.voice_key or voice.id
             if tts_provider == "openai_compatible":
@@ -131,8 +142,8 @@ def _resolve_runtime_metadata(db: Session, assistant: Assistant) -> tuple[Dict[s
                 "enabled": True,
                 "provider": tts_provider,
                 "model": model,
-                "apiKey": voice.api_key if tts_provider == "openai_compatible" else None,
-                "baseUrl": voice.base_url if tts_provider == "openai_compatible" else None,
+                "apiKey": voice.api_key if tts_provider in {"openai_compatible", "dashscope"} else None,
+                "baseUrl": voice.base_url if tts_provider in {"openai_compatible", "dashscope"} else None,
                 "voice": runtime_voice,
                 "speed": assistant.speed or voice.speed,
             }

api/app/routers/voices.py

@@ -1,6 +1,10 @@
 import base64
+import io
+import json
 import os
-from typing import Optional
+import threading
+import wave
+from typing import Any, Dict, Optional
 
 import httpx
 from fastapi import APIRouter, Depends, HTTPException
@@ -14,6 +18,203 @@ from ..schemas import VoiceCreate, VoiceOut, VoicePreviewRequest, VoicePreviewRe
 router = APIRouter(prefix="/voices", tags=["Voices"])
 
 OPENAI_COMPATIBLE_DEFAULT_MODEL = "FunAudioLLM/CosyVoice2-0.5B"
+DASHSCOPE_DEFAULT_MODEL = "qwen3-tts-flash-realtime"
+DASHSCOPE_DEFAULT_VOICE_KEY = "Cherry"
+DASHSCOPE_DEFAULT_BASE_URL = "wss://dashscope.aliyuncs.com/api-ws/v1/realtime"
+
+try:
+    import dashscope
+    from dashscope.audio.qwen_tts_realtime import AudioFormat, QwenTtsRealtime, QwenTtsRealtimeCallback
+
+    DASHSCOPE_SDK_AVAILABLE = True
+except ImportError:
+    dashscope = None  # type: ignore[assignment]
+    AudioFormat = None  # type: ignore[assignment]
+    QwenTtsRealtime = None  # type: ignore[assignment]
+    DASHSCOPE_SDK_AVAILABLE = False
+
+    class QwenTtsRealtimeCallback:  # type: ignore[no-redef]
+        """Fallback callback base when DashScope SDK is unavailable."""
+
+        pass
+
+
+class _DashScopePreviewCallback(QwenTtsRealtimeCallback):
+    """Collect DashScope realtime callback events and PCM chunks."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._open_event = threading.Event()
+        self._done_event = threading.Event()
+        self._lock = threading.Lock()
+        self._audio_chunks: list[bytes] = []
+        self._error_message: Optional[str] = None
+
+    def on_open(self) -> None:
+        self._open_event.set()
+
+    def on_close(self, code: int, reason: str) -> None:
+        if not self._done_event.is_set():
+            self._error_message = f"DashScope websocket closed unexpectedly: {code} {reason}"
+            self._done_event.set()
+
+    def on_error(self, message: str) -> None:
+        self._error_message = str(message)
+        self._done_event.set()
+
+    def on_event(self, response: Any) -> None:
+        payload = _coerce_dashscope_event(response)
+        event_type = str(payload.get("type") or "").strip()
+        if event_type == "response.audio.delta":
+            delta = payload.get("delta")
+            if isinstance(delta, str):
+                try:
+                    self._append_audio(base64.b64decode(delta))
+                except Exception:
+                    return
+        elif event_type in {"response.done", "session.finished"}:
+            self._done_event.set()
+        elif event_type == "error":
+            self._error_message = _format_dashscope_error_event(payload)
+            self._done_event.set()
+
+    def on_data(self, data: bytes) -> None:
+        # Some SDK versions emit raw PCM frames via on_data.
+        if isinstance(data, (bytes, bytearray)):
+            self._append_audio(bytes(data))
+
+    def wait_for_open(self, timeout: float = 10.0) -> None:
+        if not self._open_event.wait(timeout):
+            raise TimeoutError("DashScope websocket open timeout")
+
+    def wait_for_done(self, timeout: float = 45.0) -> None:
+        if not self._done_event.wait(timeout):
+            raise TimeoutError("DashScope synthesis timeout")
+
+    def raise_if_error(self) -> None:
+        if self._error_message:
+            raise RuntimeError(self._error_message)
+
+    def read_audio(self) -> bytes:
+        with self._lock:
+            return b"".join(self._audio_chunks)
+
+    def _append_audio(self, chunk: bytes) -> None:
+        if not chunk:
+            return
+        with self._lock:
+            self._audio_chunks.append(chunk)
+
+
+def _coerce_dashscope_event(response: Any) -> Dict[str, Any]:
+    if isinstance(response, dict):
+        return response
+    if isinstance(response, str):
+        try:
+            parsed = json.loads(response)
+            if isinstance(parsed, dict):
+                return parsed
+        except json.JSONDecodeError:
+            pass
+    return {"type": "raw", "message": str(response)}
+
+
+def _format_dashscope_error_event(payload: Dict[str, Any]) -> str:
+    error = payload.get("error")
+    if isinstance(error, dict):
+        code = str(error.get("code") or "").strip()
+        message = str(error.get("message") or "").strip()
+        if code and message:
+            return f"{code}: {message}"
+        return message or str(error)
+    return str(error or "DashScope realtime TTS error")
+
+
+def _create_dashscope_realtime_client(*, model: str, callback: _DashScopePreviewCallback, url: str, api_key: str) -> Any:
+    if QwenTtsRealtime is None:
+        raise RuntimeError("DashScope SDK unavailable")
+
+    init_kwargs = {
+        "model": model,
+        "callback": callback,
+        "url": url,
+    }
+    try:
+        return QwenTtsRealtime(api_key=api_key, **init_kwargs)  # type: ignore[misc]
+    except TypeError as exc:
+        if "api_key" not in str(exc):
+            raise
+        return QwenTtsRealtime(**init_kwargs)  # type: ignore[misc]
+
+
+def _pcm16_to_wav_bytes(pcm_bytes: bytes, sample_rate: int = 24000) -> bytes:
+    with io.BytesIO() as buffer:
+        with wave.open(buffer, "wb") as wav_file:
+            wav_file.setnchannels(1)
+            wav_file.setsampwidth(2)
+            wav_file.setframerate(sample_rate)
+            wav_file.writeframes(pcm_bytes)
+        return buffer.getvalue()
+
+
+def _synthesize_dashscope_preview(
+    *,
+    text: str,
+    api_key: str,
+    base_url: str,
+    model: str,
+    voice_key: str,
+    speed: Optional[float],
+) -> bytes:
+    if not DASHSCOPE_SDK_AVAILABLE:
+        raise RuntimeError("dashscope package not installed; install with `pip install dashscope>=1.25.11`")
+    if not AudioFormat:
+        raise RuntimeError("DashScope SDK AudioFormat unavailable")
+
+    callback = _DashScopePreviewCallback()
+    if dashscope is not None:
+        dashscope.api_key = api_key
+    client = _create_dashscope_realtime_client(
+        model=model,
+        callback=callback,
+        url=base_url,
+        api_key=api_key,
+    )
+
+    try:
+        client.connect()
+        callback.wait_for_open()
+        session_kwargs: Dict[str, Any] = {
+            "voice": voice_key,
+            "response_format": AudioFormat.PCM_24000HZ_MONO_16BIT,
+            "mode": "commit",
+        }
+        # speech_rate is supported by qwen3-* realtime models.
+        normalized_model = str(model or "").strip().lower()
+        if speed is not None and normalized_model.startswith("qwen3-"):
+            session_kwargs["speech_rate"] = max(0.5, min(2.0, float(speed)))
+        client.update_session(**session_kwargs)
+        client.append_text(text)
+        client.commit()
+        callback.wait_for_done()
+        callback.raise_if_error()
+        pcm_audio = callback.read_audio()
+        if not pcm_audio:
+            raise RuntimeError("No audio chunk returned from DashScope realtime synthesis")
+        return _pcm16_to_wav_bytes(pcm_audio, sample_rate=24000)
+    finally:
+        finish_fn = getattr(client, "finish", None)
+        if callable(finish_fn):
+            try:
+                finish_fn()
+            except Exception:
+                pass
+        close_fn = getattr(client, "close", None)
+        if callable(close_fn):
+            try:
+                close_fn()
+            except Exception:
+                pass
 
 
 def _is_openai_compatible_vendor(vendor: str) -> bool:
@@ -26,9 +227,18 @@ def _is_openai_compatible_vendor(vendor: str) -> bool:
     }
 
 
+def _is_dashscope_vendor(vendor: str) -> bool:
+    normalized = (vendor or "").strip().lower()
+    return normalized in {
+        "dashscope",
+    }
+
+
 def _default_base_url(vendor: str) -> Optional[str]:
     if _is_openai_compatible_vendor(vendor):
         return "https://api.siliconflow.cn/v1"
+    if _is_dashscope_vendor(vendor):
+        return DASHSCOPE_DEFAULT_BASE_URL
     return None
 
 
@@ -76,6 +286,9 @@ def create_voice(data: VoiceCreate, db: Session = Depends(get_db)):
         if not voice_key:
             raw_id = (data.id or data.name).strip()
             voice_key = raw_id if ":" in raw_id else f"{model}:{raw_id}"
+    elif _is_dashscope_vendor(vendor):
+        model = (model or "").strip() or DASHSCOPE_DEFAULT_MODEL
+        voice_key = (voice_key or "").strip() or DASHSCOPE_DEFAULT_VOICE_KEY
 
     voice = Voice(
         id=unique_short_id("tts", db, Voice),
@@ -126,6 +339,11 @@ def update_voice(id: str, data: VoiceUpdate, db: Session = Depends(get_db)):
         voice_key = update_data.get("voice_key") or voice.voice_key
         update_data["model"] = model
         update_data["voice_key"] = voice_key or _build_openai_compatible_voice_key(voice, model)
+    elif _is_dashscope_vendor(vendor_for_defaults):
+        model = update_data.get("model") or voice.model or DASHSCOPE_DEFAULT_MODEL
+        voice_key = update_data.get("voice_key") or voice.voice_key or DASHSCOPE_DEFAULT_VOICE_KEY
+        update_data["model"] = model
+        update_data["voice_key"] = voice_key
 
     for field, value in update_data.items():
         setattr(voice, field, value)
@@ -148,7 +366,7 @@ def delete_voice(id: str, db: Session = Depends(get_db)):
 
 @router.post("/{id}/preview", response_model=VoicePreviewResponse)
 def preview_voice(id: str, data: VoicePreviewRequest, db: Session = Depends(get_db)):
-    """试听指定声音，基于 OpenAI-compatible /audio/speech 接口。"""
+    """试听指定声音，支持 OpenAI-compatible 与 DashScope Realtime。"""
     voice = db.query(Voice).filter(Voice.id == id).first()
     if not voice:
         raise HTTPException(status_code=404, detail="Voice not found")
@@ -157,6 +375,31 @@ def preview_voice(id: str, data: VoicePreviewRequest, db: Session = Depends(get_
     if not text:
         raise HTTPException(status_code=400, detail="Preview text cannot be empty")
 
+    if _is_dashscope_vendor(voice.vendor):
+        api_key = (data.api_key or "").strip() or (voice.api_key or "").strip()
+        if not api_key:
+            api_key = os.getenv("DASHSCOPE_API_KEY", "").strip() or os.getenv("TTS_API_KEY", "").strip()
+        if not api_key:
+            raise HTTPException(status_code=400, detail=f"API key is required for voice: {voice.name}")
+
+        base_url = (voice.base_url or "").strip() or DASHSCOPE_DEFAULT_BASE_URL
+        model = (voice.model or "").strip() or DASHSCOPE_DEFAULT_MODEL
+        voice_key = (voice.voice_key or "").strip() or DASHSCOPE_DEFAULT_VOICE_KEY
+        effective_speed = data.speed if data.speed is not None else voice.speed
+        try:
+            wav_bytes = _synthesize_dashscope_preview(
+                text=text,
+                api_key=api_key,
+                base_url=base_url,
+                model=model,
+                voice_key=voice_key,
+                speed=effective_speed,
+            )
+        except Exception as exc:
+            raise HTTPException(status_code=502, detail=f"DashScope preview failed: {exc}") from exc
+        audio_base64 = base64.b64encode(wav_bytes).decode("utf-8")
+        return VoicePreviewResponse(success=True, audio_url=f"data:audio/wav;base64,{audio_base64}")
+
     api_key = (data.api_key or "").strip() or (voice.api_key or "").strip()
     if not api_key and _is_openai_compatible_vendor(voice.vendor):
         api_key = os.getenv("SILICONFLOW_API_KEY", "").strip()