Add model-specific sample rates, transport example, and fix audio buffer alignment

This commit is contained in:
Neil Ruaro
2026-01-13 06:32:11 +09:00
parent e76a3d04f0
commit ed120d014d
4 changed files with 176 additions and 39 deletions

View File

@@ -10,11 +10,10 @@ This module provides TTS functionality using Camb.ai's MARS model family,
offering high-quality text-to-speech synthesis with streaming support.
Features:
- MARS models: mars-flash, mars-pro, mars-instruct
- MARS models: mars-flash (fast), mars-pro (high quality)
- 140+ languages supported
- Real-time streaming via official SDK
- 48kHz audio output
- Voice customization (instructions for mars-instruct)
- Model-specific sample rates: mars-pro (48kHz), mars-flash (22.05kHz)
"""
from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional
@@ -41,11 +40,17 @@ from pipecat.utils.tracing.service_decorators import traced_tts
DEFAULT_VOICE_ID = 147320
DEFAULT_LANGUAGE = "en-us"
DEFAULT_MODEL = "mars-flash" # Faster inference
DEFAULT_SAMPLE_RATE = 48000 # 48kHz
DEFAULT_TIMEOUT = 60.0 # Seconds (minimum recommended by Camb.ai)
MIN_TEXT_LENGTH = 3
MAX_TEXT_LENGTH = 3000
# Model-specific sample rates
MODEL_SAMPLE_RATES: Dict[str, int] = {
"mars-flash": 22050, # 22.05kHz
"mars-pro": 48000, # 48kHz
"mars-instruct": 22050, # 22.05kHz
}
# Gender mapping for voice listing
GENDER_MAP = {0: "Not Specified", 1: "Male", 2: "Female", 9: "Not Applicable"}
@@ -131,30 +136,23 @@ class CambTTSService(TTSService):
"""Camb.ai MARS text-to-speech service using the official SDK.
Converts text to speech using Camb.ai's MARS TTS models with support for
multiple languages. Provides custom instructions support for the mars-instruct model.
multiple languages.
All models output 48kHz audio.
Models:
- mars-flash: Fast inference, 22.05kHz output (default)
- mars-pro: High quality, 48kHz output
Example::
# Basic usage with defaults
# Basic usage with defaults (mars-flash)
tts = CambTTSService(api_key="your-api-key")
# With custom voice and model
# High quality with mars-pro
tts = CambTTSService(
api_key="your-api-key",
voice_id=12345,
model="mars-pro",
)
# mars-instruct with custom instructions
tts = CambTTSService(
api_key="your-api-key",
model="mars-instruct",
params=CambTTSService.InputParams(
user_instructions="Speak with excitement and energy"
),
)
"""
class InputParams(BaseModel):
@@ -190,10 +188,10 @@ class CambTTSService(TTSService):
Args:
api_key: Camb.ai API key for authentication.
voice_id: Voice ID to use. Defaults to DEFAULT_VOICE_ID.
model: TTS model to use. Options: "mars-flash", "mars-pro", "mars-instruct".
Defaults to DEFAULT_MODEL (mars-flash, fastest).
model: TTS model to use. Options: "mars-flash" (fast), "mars-pro" (high quality).
Defaults to DEFAULT_MODEL (mars-flash).
timeout: Request timeout in seconds. Defaults to DEFAULT_TIMEOUT (60s).
sample_rate: Audio sample rate in Hz. If None, uses DEFAULT_SAMPLE_RATE (48kHz).
sample_rate: Audio sample rate in Hz. If None, uses model-specific default.
params: Additional voice parameters. If None, uses defaults.
**kwargs: Additional arguments passed to parent TTSService.
"""
@@ -243,9 +241,9 @@ class CambTTSService(TTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
# Use 48kHz sample rate if not explicitly specified
# Use model-specific sample rate if not explicitly specified
if not self._init_sample_rate:
self._sample_rate = DEFAULT_SAMPLE_RATE
self._sample_rate = MODEL_SAMPLE_RATES.get(self._model_name, 22050)
self._settings["sample_rate"] = self._sample_rate
async def _update_settings(self, settings: Mapping[str, Any]):
@@ -310,15 +308,33 @@ class CambTTSService(TTSService):
await self.start_tts_usage_metrics(text)
yield TTSStartedFrame()
# Buffer for aligning chunks to 2-byte boundaries (16-bit PCM)
audio_buffer = b""
# Stream audio chunks from SDK
async for chunk in self._client.text_to_speech.tts(**tts_kwargs):
if chunk:
await self.stop_ttfb_metrics()
yield TTSAudioRawFrame(
audio=chunk,
sample_rate=self.sample_rate,
num_channels=1,
)
audio_buffer += chunk
# Only yield complete 16-bit samples (2 bytes per sample)
aligned_size = (len(audio_buffer) // 2) * 2
if aligned_size > 0:
yield TTSAudioRawFrame(
audio=audio_buffer[:aligned_size],
sample_rate=self.sample_rate,
num_channels=1,
)
audio_buffer = audio_buffer[aligned_size:]
# Yield any remaining complete samples
if len(audio_buffer) >= 2:
aligned_size = (len(audio_buffer) // 2) * 2
yield TTSAudioRawFrame(
audio=audio_buffer[:aligned_size],
sample_rate=self.sample_rate,
num_channels=1,
)
except Exception as e:
error_msg = f"Camb.ai TTS error: {e}"