wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(minimax): comprehensive updates to TTS service

Browse Source 
- Add support for speech-2.6-hd and speech-2.6-turbo models
- Add 16 new languages (total 40): Afrikaans, Bulgarian, Catalan, Danish, Persian, Filipino, Hebrew, Croatian, Hungarian, Malay, Norwegian, Nynorsk, Slovak, Slovenian, Swedish, Tamil
- Add new emotions: calm and fluent
- Add new parameters: text_normalization (renamed from english_normalization), latex_read, force_cbr, exclude_aggregated_audio, subtitle_enable, subtitle_type
- Extract trace_id from response headers for all requests
- Improve error handling for non-streaming error responses
- Add detailed extra_info logging (audio_length, audio_size, usage_characters, word_count)
- Add validation warnings for language/model compatibility
- Fix silent error issue where HTTP 200 responses with errors were ignored

BREAKING CHANGE: Renamed parameter english_normalization to text_normalization
This commit is contained in:
minimax
2025-11-04 02:04:25 +08:00
committed by vipyne
parent 510f3df6b7
commit 77be2bd013
2 changed files with 252 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/pipecat/__init__.py
View File

@@ -5,11 +5,14 @@
#
import sys
from importlib.metadata import version
from importlib.metadata import version, PackageNotFoundError
from loguru import logger
__version__ = version("pipecat-ai")
try:
__version__ = version("pipecat-ai")
except PackageNotFoundError:
__version__ = "0.0.0.dev" # Development version
logger.info(f"ᓚᘏᗢ Pipecat {__version__} (Python {sys.version}) ᓚᘏᗢ")

269
src/pipecat/services/minimax/tts.py
View File

@@ -40,24 +40,40 @@ def language_to_minimax_language(language: Language) -> Optional[str]:
The corresponding MiniMax language name, or None if not supported.
"""
LANGUAGE_MAP = {
Language.AF: "Afrikaans",
Language.AR: "Arabic",
Language.BG: "Bulgarian",
Language.CA: "Catalan",
Language.CS: "Czech",
Language.DA: "Danish",
Language.DE: "German",
Language.EL: "Greek",
Language.EN: "English",
Language.ES: "Spanish",
Language.FA: "Persian", # ⚠️ Only supported by speech-2.6-* models
Language.FI: "Finnish",
Language.FIL: "Filipino", # ⚠️ Only supported by speech-2.6-* models
Language.FR: "French",
Language.HE: "Hebrew",
Language.HI: "Hindi",
Language.HR: "Croatian",
Language.HU: "Hungarian",
Language.ID: "Indonesian",
Language.IT: "Italian",
Language.JA: "Japanese",
Language.KO: "Korean",
Language.MS: "Malay",
Language.NB: "Norwegian",
Language.NN: "Nynorsk",
Language.NL: "Dutch",
Language.PL: "Polish",
Language.PT: "Portuguese",
Language.RO: "Romanian",
Language.RU: "Russian",
Language.SK: "Slovak",
Language.SL: "Slovenian",
Language.SV: "Swedish",
Language.TA: "Tamil", # ⚠️ Only supported by speech-2.6-* models
Language.TH: "Thai",
Language.TR: "Turkish",
Language.UK: "Ukrainian",
@@ -65,6 +81,9 @@ def language_to_minimax_language(language: Language) -> Optional[str]:
Language.YUE: "Chinese,Yue",
Language.ZH: "Chinese",
}
# Languages that require speech-2.6-* models
V26_ONLY_LANGUAGES = {Language.FA, Language.FIL, Language.TA}
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
@@ -84,13 +103,20 @@ class MiniMaxHttpTTSService(TTSService):
"""Configuration parameters for MiniMax TTS.
Parameters:
language: Language for TTS generation.
language: Language for TTS generation. Supports 40 languages.
Note: Filipino, Tamil, and Persian require speech-2.6-* models.
speed: Speech speed (range: 0.5 to 2.0).
volume: Speech volume (range: 0 to 10).
pitch: Pitch adjustment (range: -12 to 12).
emotion: Emotional tone (options: "happy", "sad", "angry", "fearful",
"disgusted", "surprised", "neutral").
english_normalization: Whether to apply English text normalization.
"disgusted", "surprised", "calm", "fluent").
text_normalization: Enable text normalization (Chinese/English).
latex_read: Enable LaTeX formula reading.
force_cbr: Enable Constant Bitrate (CBR) for audio encoding (MP3 only).
exclude_aggregated_audio: Whether to exclude aggregated audio in final chunk.
subtitle_enable: Enable subtitle generation (non-streaming only).
subtitle_type: Subtitle timestamp granularity (options: "word", "sentence").
Only effective when subtitle_enable is True. Defaults to "sentence".
"""
language: Optional[Language] = Language.EN
@@ -98,7 +124,12 @@ class MiniMaxHttpTTSService(TTSService):
volume: Optional[float] = 1.0
pitch: Optional[int] = 0
emotion: Optional[str] = None
english_normalization: Optional[bool] = None
text_normalization: Optional[bool] = None
latex_read: Optional[bool] = None
force_cbr: Optional[bool] = None
exclude_aggregated_audio: Optional[bool] = None
subtitle_enable: Optional[bool] = None
subtitle_type: Optional[str] = "sentence"
def __init__(
self,
@@ -121,8 +152,10 @@ class MiniMaxHttpTTSService(TTSService):
Global: https://api.minimax.io/v1/t2a_v2
Mainland China: https://api.minimaxi.chat/v1/t2a_v2
group_id: MiniMax Group ID to identify project.
model: TTS model name. Defaults to "speech-02-turbo". Options include
"speech-02-hd", "speech-02-turbo", "speech-01-hd", "speech-01-turbo".
model: TTS model name. Defaults to "speech-02-turbo". Options include:
"speech-2.6-hd", "speech-2.6-turbo" (latest, supports Filipino/Tamil/Persian),
"speech-02-hd", "speech-02-turbo",
"speech-01-hd", "speech-01-turbo".
voice_id: Voice identifier. Defaults to "Calm_Woman".
aiohttp_session: aiohttp.ClientSession for API communication.
sample_rate: Output audio sample rate in Hz. If None, uses pipeline default.
@@ -139,6 +172,7 @@ class MiniMaxHttpTTSService(TTSService):
self._session = aiohttp_session
self._model_name = model
self._voice_id = voice_id
self._current_trace_id: Optional[str] = None
# Create voice settings
self._settings = {
@@ -155,6 +189,12 @@ class MiniMaxHttpTTSService(TTSService):
},
}
# Add stream_options if exclude_aggregated_audio is set
if params.exclude_aggregated_audio is not None:
self._settings["stream_options"] = {
"exclude_aggregated_audio": params.exclude_aggregated_audio
}
# Set voice and model
self.set_voice(voice_id)
self.set_model_name(model)
@@ -164,10 +204,21 @@ class MiniMaxHttpTTSService(TTSService):
service_lang = self.language_to_service_language(params.language)
if service_lang:
self._settings["language_boost"] = service_lang
# Validate language-model compatibility
# Filipino, Tamil, Persian only supported by speech-2.6-* models
if params.language in {Language.FA, Language.FIL, Language.TA}:
if not model.startswith("speech-2.6"):
logger.warning(
f"Language {params.language.value} ({service_lang}) is only supported by "
f"speech-2.6-hd and speech-2.6-turbo models. "
f"Current model '{model}' may not support this language. "
f"Consider using 'speech-2.6-turbo' or 'speech-2.6-hd'."
)
# Add optional emotion if provided
if params.emotion:
# Validate emotion is in the supported list
# Validate emotion is in the supported list (updated per official docs)
supported_emotions = [
"happy",
"sad",
@@ -176,15 +227,58 @@ class MiniMaxHttpTTSService(TTSService):
"disgusted",
"surprised",
"neutral",
"fluent",
]
if params.emotion in supported_emotions:
self._settings["voice_setting"]["emotion"] = params.emotion
else:
logger.warning(f"Unsupported emotion: {params.emotion}. Using default.")
logger.warning(
f"Unsupported emotion: {params.emotion}. Supported emotions: {supported_emotions}"
)
# Add english_normalization if provided
if params.english_normalization is not None:
self._settings["english_normalization"] = params.english_normalization
# Add text_normalization if provided (corrected parameter name)
if params.text_normalization is not None:
self._settings["voice_setting"]["text_normalization"] = params.text_normalization
# Add latex_read if provided
if params.latex_read is not None:
self._settings["voice_setting"]["latex_read"] = params.latex_read
# Add force_cbr if provided (for MP3 format only)
if params.force_cbr is not None:
self._settings["audio_setting"]["force_cbr"] = params.force_cbr
# Add subtitle settings if provided
if params.subtitle_enable is not None:
# Note: subtitle_enable only works in non-streaming mode (stream=false)
# Current implementation uses streaming mode (stream=true)
if params.subtitle_enable:
logger.warning(
"subtitle_enable is set to True, but this service uses streaming mode. "
"Subtitle generation (subtitle_enable) only works in non-streaming mode. "
"Subtitles may not be generated. "
"For subtitle support, consider implementing a non-streaming TTS service."
)
self._settings["subtitle_enable"] = params.subtitle_enable
# Add subtitle_type only when subtitle_enable is True
if params.subtitle_enable and params.subtitle_type:
# Validate subtitle_type
if params.subtitle_type not in ["word", "sentence"]:
logger.warning(
f"Invalid subtitle_type: {params.subtitle_type}. "
f"Must be 'word' or 'sentence'. Using default 'sentence'."
)
self._settings["subtitle_type"] = "sentence"
else:
self._settings["subtitle_type"] = params.subtitle_type
elif not params.subtitle_enable and params.subtitle_type != "sentence":
# Warn if subtitle_type is set but subtitle_enable is False
logger.debug(
f"subtitle_type='{params.subtitle_type}' will be ignored because "
f"subtitle_enable is False."
)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -231,7 +325,7 @@ class MiniMaxHttpTTSService(TTSService):
"""
await super().start(frame)
self._settings["audio_setting"]["sample_rate"] = self.sample_rate
logger.debug(f"MiniMax TTS initialized with sample rate: {self.sample_rate}")
logger.debug(f"MiniMax TTS initialized with sample_rate={self.sample_rate}")
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
@@ -245,6 +339,9 @@ class MiniMaxHttpTTSService(TTSService):
"""
logger.debug(f"{self}: Generating TTS [{text}]")
# Reset trace_id for new request
self._current_trace_id = None
headers = {
"accept": "application/json, text/plain, */*",
"Content-Type": "application/json",
@@ -262,9 +359,46 @@ class MiniMaxHttpTTSService(TTSService):
async with self._session.post(
self._base_url, headers=headers, json=payload
) as response:
# Extract trace_id from response header (available in all responses)
trace_id = response.headers.get("Trace-Id") or response.headers.get("trace-id") or response.headers.get("X-Trace-Id") or "unknown"
self._current_trace_id = trace_id
# Log trace_id for all requests
logger.info(f"MiniMax TTS request trace_id={trace_id}, status={response.status}")
if response.status != 200:
error_message = f"MiniMax TTS error: HTTP {response.status}"
logger.error(error_message)
# Try to read error response body
try:
error_body = await response.text()
error_data = json.loads(error_body)
# Extract MiniMax error details from body
base_resp = error_data.get("base_resp", {})
status_code = base_resp.get("status_code", response.status)
status_msg = base_resp.get("status_msg", "Unknown error")
error_message = (
f"MiniMax TTS error: HTTP {response.status}, "
f"status_code={status_code}, status_msg={status_msg}, "
f"trace_id={trace_id}"
)
logger.error(
error_message,
extra={
"trace_id": trace_id,
"http_status": response.status,
"status_code": status_code,
"text_length": len(text),
},
)
except Exception as parse_error:
# If parsing fails, use basic error message
error_message = f"MiniMax TTS error: HTTP {response.status}, trace_id={trace_id}"
logger.error(
error_message,
extra={"http_status": response.status, "trace_id": trace_id, "parse_error": str(parse_error)},
)
yield ErrorFrame(error=error_message)
return
@@ -272,15 +406,49 @@ class MiniMaxHttpTTSService(TTSService):
yield TTSStartedFrame()
# Process the streaming response
logger.debug(f"Starting to read streaming response, status={response.status}, trace_id={trace_id}")
buffer = bytearray()
CHUNK_SIZE = self.chunk_size
chunk_count = 0
async for chunk in response.content.iter_chunked(CHUNK_SIZE):
chunk_count += 1
logger.debug(f"Received chunk #{chunk_count}, size={len(chunk)} bytes")
if not chunk:
continue
buffer.extend(chunk)
# Log raw buffer content for debugging
if chunk_count == 1:
logger.debug(f"Raw buffer content: {buffer[:200]}") # First 200 bytes
# Check if first chunk is a direct JSON error (not streaming format)
if not buffer.startswith(b"data:"):
try:
error_data = json.loads(buffer.decode("utf-8"))
base_resp = error_data.get("base_resp", {})
status_code = base_resp.get("status_code", 0)
if status_code != 0:
# This is a non-streaming error response
# Use trace_id from header (already extracted above)
status_msg = base_resp.get("status_msg", "Unknown error")
error_message = (
f"MiniMax TTS API error: status_code={status_code}, "
f"status_msg={status_msg}, trace_id={self._current_trace_id}"
)
logger.error(
error_message,
extra={"trace_id": self._current_trace_id, "status_code": status_code},
)
yield ErrorFrame(error=error_message)
return
except (json.JSONDecodeError, UnicodeDecodeError):
# Not a valid JSON, continue with streaming processing
pass
# Find complete data blocks
while b"data:" in buffer:
@@ -298,16 +466,56 @@ class MiniMaxHttpTTSService(TTSService):
buffer = buffer[next_start:]
try:
data = json.loads(data_block[5:].decode("utf-8"))
# Skip data blocks containing extra_info
if "extra_info" in data:
logger.debug("Received final chunk with extra info")
continue
data_str = data_block[5:].decode("utf-8")
logger.debug(f"Parsing data block: {data_str[:200]}...") # Log first 200 chars
data = json.loads(data_str)
# Check for business errors in base_resp
base_resp = data.get("base_resp", {})
status_code = base_resp.get("status_code", 0)
if status_code != 0:
# API returned business error
status_msg = base_resp.get("status_msg", "Unknown error")
error_message = (
f"MiniMax TTS API error: status_code={status_code}, "
f"status_msg={status_msg}, trace_id={self._current_trace_id}"
)
logger.error(
error_message,
extra={"trace_id": self._current_trace_id, "status_code": status_code},
)
yield ErrorFrame(error=error_message)
return
# Handle final chunk with extra_info
if "extra_info" in data:
extra_info = data.get("extra_info", {})
logger.info(
f"MiniMax TTS completed successfully, trace_id={self._current_trace_id}",
extra={
"trace_id": self._current_trace_id,
"audio_length": extra_info.get("audio_length"),
"audio_size": extra_info.get("audio_size"),
"usage_characters": extra_info.get("usage_characters"),
"word_count": extra_info.get("word_count"),
},
)
continue # No audio data in this block
# Extract audio data
chunk_data = data.get("data", {})
if not chunk_data:
continue
# Check for subtitle file (if subtitle generation is enabled)
subtitle_file = chunk_data.get("subtitle_file")
if subtitle_file:
logger.info(
f"Subtitle file available: {subtitle_file}",
extra={"trace_id": self._current_trace_id, "subtitle_url": subtitle_file},
)
audio_data = chunk_data.get("audio")
if not audio_data:
continue
@@ -320,7 +528,7 @@ class MiniMaxHttpTTSService(TTSService):
continue
try:
# Convert this chunk of data
# Convert hex to binary
audio_chunk = bytes.fromhex(hex_chunk)
if audio_chunk:
await self.stop_ttfb_metrics()
@@ -330,16 +538,33 @@ class MiniMaxHttpTTSService(TTSService):
num_channels=1,
)
except ValueError as e:
logger.error(f"Error converting hex to binary: {e}")
logger.error(
f"Error converting hex to binary: {e}",
extra={"trace_id": self._current_trace_id},
)
continue
except json.JSONDecodeError as e:
logger.error(f"Error decoding JSON: {e}, data: {data_block[:100]}")
logger.error(
f"Error decoding JSON: {e}, data: {data_block[:100]}",
extra={"trace_id": self._current_trace_id or "unknown"},
)
continue
except aiohttp.ClientError as e:
error_msg = f"MiniMax TTS network error: {str(e)}"
logger.exception(error_msg, extra={"trace_id": self._current_trace_id or "unknown"})
yield ErrorFrame(error=error_msg)
except Exception as e:
logger.error(f"{self} exception: {e}")
yield ErrorFrame(error=f"{self} error: {e}")
finally:
if self._current_trace_id:
logger.debug(
f"MiniMax TTS request finished, trace_id={self._current_trace_id}, "
f"received {chunk_count if 'chunk_count' in locals() else 0} chunks"
)
else:
logger.debug(f"MiniMax TTS request finished with no trace_id (no data received)")
await self.stop_ttfb_metrics()
yield TTSStoppedFrame()