From c3d8ea210f9dc2006a1fd41ff6d5eba59ef66171 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 31 Jul 2025 00:16:00 -0400 Subject: [PATCH] CartesiaTTSService: Add CJK lang support for word timestamps --- CHANGELOG.md | 3 ++ src/pipecat/services/cartesia/tts.py | 54 ++++++++++++++++++++++++++-- 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 658218361..e3b4fdc6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 average response time between user stop and bot start, as well as minimum and maximum response latency. +- Added Chinese, Japanese, Korean word timestamp support to + `CartesiaTTSService`. + ### Changed - Updated the `pipecat.runner.daily` utility to only a take `DAILY_API_URL` and diff --git a/src/pipecat/services/cartesia/tts.py b/src/pipecat/services/cartesia/tts.py index c731a6984..1d136feef 100644 --- a/src/pipecat/services/cartesia/tts.py +++ b/src/pipecat/services/cartesia/tts.py @@ -216,6 +216,54 @@ class CartesiaTTSService(AudioContextWordTTSService): """ return language_to_cartesia_language(language) + def _is_cjk_language(self, language: str) -> bool: + """Check if the given language is CJK (Chinese, Japanese, Korean). + + Args: + language: The language code to check. + + Returns: + True if the language is Chinese, Japanese, or Korean. + """ + cjk_languages = {"zh", "ja", "ko"} + base_lang = language.split("-")[0].lower() + return base_lang in cjk_languages + + def _process_word_timestamps_for_language( + self, words: List[str], starts: List[float] + ) -> List[tuple[str, float]]: + """Process word timestamps based on the current language. + + For CJK languages, Cartesia groups related characters in the same timestamp message. + For example, in Japanese a single message might be `['こ', 'ん', 'に', 'ち', 'は', '。']`. + We combine these into single words so the downstream aggregator can add natural + spacing between meaningful units rather than individual characters. + + For non-CJK languages, words are already properly separated and are used as-is. + + Args: + words: List of words/characters from Cartesia. + starts: List of start timestamps for each word/character. + + Returns: + List of (word, start_time) tuples processed for the language. + """ + current_language = self._settings.get("language", "en") + + # Check if this is a CJK language + if self._is_cjk_language(current_language): + # For CJK languages, combine all characters in this message into one word + # using the first character's start time + if words and starts: + combined_word = "".join(words) + first_start = starts[0] + return [(combined_word, first_start)] + else: + return [] + else: + # For non-CJK languages, use as-is + return list(zip(words, starts)) + def _build_msg( self, text: str = "", continue_transcript: bool = True, add_timestamps: bool = True ): @@ -351,9 +399,11 @@ class CartesiaTTSService(AudioContextWordTTSService): await self.add_word_timestamps([("TTSStoppedFrame", 0), ("Reset", 0)]) await self.remove_audio_context(msg["context_id"]) elif msg["type"] == "timestamps": - await self.add_word_timestamps( - list(zip(msg["word_timestamps"]["words"], msg["word_timestamps"]["start"])) + # Process the timestamps based on language before adding them + processed_timestamps = self._process_word_timestamps_for_language( + msg["word_timestamps"]["words"], msg["word_timestamps"]["start"] ) + await self.add_word_timestamps(processed_timestamps) elif msg["type"] == "chunk": await self.stop_ttfb_metrics() self.start_word_timestamps()