wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #3642 from pipecat-ai/cb/rime-arcana-v3

Browse Source 
Update RimeTTSService for arcana and mistv2 model support
This commit is contained in:
Mark Backman
2026-02-13 13:25:27 -05:00
committed by GitHub
parent ca440594fe 18afe37bd1
commit 79ec61d1d8
4 changed files with 144 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
changelog/3642.added.md Normal file
View File

@@ -0,0 +1 @@
- Added model-specific `InputParams` to `RimeTTSService`: arcana params (`repetition_penalty`, `temperature`, `top_p`) and mistv2 params (`no_text_normalization`, `save_oovs`, `segment`). Model, voice, and param changes now trigger WebSocket reconnection.

1
changelog/3642.changed.md Normal file
View File

@@ -0,0 +1 @@
- ⚠️ `RimeTTSService` now defaults to `model="arcana"` and the `wss://users-ws.rime.ai/ws3` endpoint. `InputParams` defaults changed from mistv2-specific values to `None` — only explicitly-set params are sent as query params.

2
examples/foundational/07q-interruptible-rime.py
View File

@@ -56,7 +56,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
tts = RimeTTSService(
api_key=os.getenv("RIME_API_KEY", ""),
voice_id="rex",
voice_id="luna",
)
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))

180
src/pipecat/services/rime/tts.py
View File

@@ -82,25 +82,39 @@ class RimeTTSService(AudioContextWordTTSService):
Parameters:
language: Language for synthesis. Defaults to English.
speed_alpha: Speech speed multiplier. Defaults to 1.0.
reduce_latency: Whether to reduce latency at potential quality cost.
pause_between_brackets: Whether to add pauses between bracketed content.
phonemize_between_brackets: Whether to phonemize bracketed content.
segment: Text segmentation mode ("immediate", "bySentence", "never").
repetition_penalty: Token repetition penalty (arcana only).
temperature: Sampling temperature (arcana only).
top_p: Cumulative probability threshold (arcana only).
speed_alpha: Speech speed multiplier (mistv2 only).
reduce_latency: Whether to reduce latency at potential quality cost (mistv2 only).
pause_between_brackets: Whether to add pauses between bracketed content (mistv2 only).
phonemize_between_brackets: Whether to phonemize bracketed content (mistv2 only).
no_text_normalization: Whether to disable text normalization (mistv2 only).
save_oovs: Whether to save out-of-vocabulary words (mistv2 only).
"""
language: Optional[Language] = Language.EN
speed_alpha: Optional[float] = 1.0
reduce_latency: Optional[bool] = False
pause_between_brackets: Optional[bool] = False
phonemize_between_brackets: Optional[bool] = False
segment: Optional[str] = None
# Arcana params
repetition_penalty: Optional[float] = None
temperature: Optional[float] = None
top_p: Optional[float] = None
# Mistv2 params
speed_alpha: Optional[float] = None
reduce_latency: Optional[bool] = None
pause_between_brackets: Optional[bool] = None
phonemize_between_brackets: Optional[bool] = None
no_text_normalization: Optional[bool] = None
save_oovs: Optional[bool] = None
def __init__(
self,
*,
api_key: str,
voice_id: str,
url: str = "wss://users.rime.ai/ws2",
model: str = "mistv2",
url: str = "wss://users-ws.rime.ai/ws3",
model: str = "arcana",
sample_rate: Optional[int] = None,
params: Optional[InputParams] = None,
text_aggregator: Optional[BaseTextAggregator] = None,
@@ -143,26 +157,14 @@ class RimeTTSService(AudioContextWordTTSService):
# and insert these tags for the purpose of the TTS service alone.
self._text_aggregator = SkipTagsAggregator([("spell(", ")")])
params = params or RimeTTSService.InputParams()
self._params = params or RimeTTSService.InputParams()
# Store service configuration
self._api_key = api_key
self._url = url
self._voice_id = voice_id
self._model = model
self._settings = {
"speaker": voice_id,
"modelId": model,
"audioFormat": "pcm",
"samplingRate": 0,
"lang": self.language_to_service_language(params.language)
if params.language
else "eng",
"speedAlpha": params.speed_alpha,
"reduceLatency": params.reduce_latency,
"pauseBetweenBrackets": json.dumps(params.pause_between_brackets),
"phonemizeBetweenBrackets": json.dumps(params.phonemize_between_brackets),
}
self._settings = self._build_settings()
# State tracking
self._context_id = None # Tracks current turn
@@ -189,14 +191,60 @@ class RimeTTSService(AudioContextWordTTSService):
"""
return language_to_rime_language(language)
def _build_settings(self) -> dict:
"""Build query params for the WebSocket URL based on the current model and params.
Returns:
Dictionary of query parameters. Only explicitly-set values are included.
"""
settings = {
"speaker": self._voice_id,
"modelId": self._model,
"audioFormat": "pcm",
"samplingRate": self.sample_rate or 0,
}
if self._params.language:
settings["lang"] = self.language_to_service_language(self._params.language) or "eng"
if self._params.segment is not None:
settings["segment"] = self._params.segment
if self._model == "arcana":
if self._params.repetition_penalty is not None:
settings["repetition_penalty"] = self._params.repetition_penalty
if self._params.temperature is not None:
settings["temperature"] = self._params.temperature
if self._params.top_p is not None:
settings["top_p"] = self._params.top_p
else: # mistv2/mist
if self._params.speed_alpha is not None:
settings["speedAlpha"] = self._params.speed_alpha
if self._params.reduce_latency is not None:
settings["reduceLatency"] = self._params.reduce_latency
if self._params.pause_between_brackets is not None:
settings["pauseBetweenBrackets"] = json.dumps(self._params.pause_between_brackets)
if self._params.phonemize_between_brackets is not None:
settings["phonemizeBetweenBrackets"] = json.dumps(
self._params.phonemize_between_brackets
)
if self._params.no_text_normalization is not None:
settings["noTextNormalization"] = json.dumps(self._params.no_text_normalization)
if self._params.save_oovs is not None:
settings["saveOovs"] = json.dumps(self._params.save_oovs)
return settings
async def set_model(self, model: str):
"""Update the TTS model.
"""Update the TTS model and reconnect.
Args:
model: The model name to use for synthesis.
"""
self._model = model
self._settings = self._build_settings()
await super().set_model(model)
if self._websocket:
await self._disconnect()
await self._connect()
# A set of Rime-specific helpers for text transformations
def SPELL(text: str) -> str:
@@ -224,12 +272,68 @@ class RimeTTSService(AudioContextWordTTSService):
return f"[{text}]"
async def _update_settings(self, settings: Mapping[str, Any]):
"""Update service settings and reconnect if voice changed."""
prev_voice = self._voice_id
"""Update service settings and reconnect if necessary.
Since all settings are WebSocket URL query parameters,
any setting change requires reconnecting to apply the new values.
"""
prev_settings = self._settings.copy()
await super()._update_settings(settings)
if not prev_voice == self._voice_id:
needs_reconnect = False
if "voice" in settings or "voice_id" in settings:
self._settings["speaker"] = self._voice_id
logger.info(f"Switching TTS voice to: [{self._voice_id}]")
if prev_settings.get("speaker") != self._voice_id:
logger.info(f"Switching TTS voice to: [{self._voice_id}]")
needs_reconnect = True
if "model" in settings:
self._settings = self._build_settings()
needs_reconnect = True
if "language" in settings:
new_lang = self.language_to_service_language(settings["language"])
if new_lang and new_lang != prev_settings.get("lang"):
logger.info(f"Updating language to: [{new_lang}]")
self._settings["lang"] = new_lang
needs_reconnect = True
# Arcana params
for key, settings_key in [
("repetition_penalty", "repetition_penalty"),
("temperature", "temperature"),
("top_p", "top_p"),
]:
if key in settings and settings[key] != prev_settings.get(settings_key):
self._settings[settings_key] = settings[key]
needs_reconnect = True
# Mistv2 params
for key, settings_key in [
("speed_alpha", "speedAlpha"),
("reduce_latency", "reduceLatency"),
]:
if key in settings and settings[key] != prev_settings.get(settings_key):
self._settings[settings_key] = settings[key]
needs_reconnect = True
# Mistv2 boolean params (need json.dumps)
for key, settings_key in [
("pause_between_brackets", "pauseBetweenBrackets"),
("phonemize_between_brackets", "phonemizeBetweenBrackets"),
("no_text_normalization", "noTextNormalization"),
("save_oovs", "saveOovs"),
]:
if key in settings and json.dumps(settings[key]) != prev_settings.get(settings_key):
self._settings[settings_key] = json.dumps(settings[key])
needs_reconnect = True
if "segment" in settings and settings["segment"] != prev_settings.get("segment"):
self._settings["segment"] = settings["segment"]
needs_reconnect = True
if needs_reconnect and self._websocket:
await self._disconnect()
await self._connect()
@@ -256,7 +360,7 @@ class RimeTTSService(AudioContextWordTTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["samplingRate"] = self.sample_rate
self._settings = self._build_settings()
await self._connect()
async def stop(self, frame: EndFrame):
@@ -302,7 +406,7 @@ class RimeTTSService(AudioContextWordTTSService):
if self._websocket and self._websocket.state is State.OPEN:
return
params = "&".join(f"{k}={v}" for k, v in self._settings.items())
params = "&".join(f"{k}={v}" for k, v in self._settings.items() if v is not None)
url = f"{self._url}?{params}"
headers = {"Authorization": f"Bearer {self._api_key}"}
self._websocket = await websocket_connect(url, additional_headers=headers)
@@ -398,7 +502,7 @@ class RimeTTSService(AudioContextWordTTSService):
async for message in self._get_websocket():
msg = json.loads(message)
if not msg or not self.audio_context_available(msg["contextId"]):
if not msg or not self.audio_context_available(msg.get("contextId")):
continue
context_id = msg["contextId"]
@@ -641,20 +745,18 @@ class RimeHttpTTSService(TTSService):
class RimeNonJsonTTSService(InterruptibleTTSService):
"""Pipecat TTS service for Rime's non-JSON WebSocket API.
.. deprecated:: 0.0.102
Arcana now supports JSON WebSocket with word-level timestamps via the
``wss://users-ws.rime.ai/ws3`` endpoint. Use :class:`RimeTTSService`
with ``model="arcana"`` instead.
This service enables Text-to-Speech synthesis over WebSocket endpoints
that require plain text (not JSON) messages and return raw audio bytes.
It is designed for use with TTS models like Arcana, which currently do
not support JSON-based WebSocket protocols (though this may change in
the future).
Limitations:
- Does not support word-level timestamps or context IDs.
- Intended specifically for integrations where the TTS provider only
accepts and returns non-JSON messages.
Note:
- Arcana and similar models may add JSON WebSocket support in the
future. This service focuses on the current plain text protocol.
"""
class InputParams(BaseModel):