diff --git a/CHANGELOG.md b/CHANGELOG.md index 024bd10f0..30c87aa5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 average response time between user stop and bot start, as well as minimum and maximum response latency. +- Added Chinese, Japanese, Korean word timestamp support to + `CartesiaTTSService`. + ### Changed - Changed the default model for both `ElevenLabsTTSService` and diff --git a/src/pipecat/services/cartesia/tts.py b/src/pipecat/services/cartesia/tts.py index c731a6984..1d136feef 100644 --- a/src/pipecat/services/cartesia/tts.py +++ b/src/pipecat/services/cartesia/tts.py @@ -216,6 +216,54 @@ class CartesiaTTSService(AudioContextWordTTSService): """ return language_to_cartesia_language(language) + def _is_cjk_language(self, language: str) -> bool: + """Check if the given language is CJK (Chinese, Japanese, Korean). + + Args: + language: The language code to check. + + Returns: + True if the language is Chinese, Japanese, or Korean. + """ + cjk_languages = {"zh", "ja", "ko"} + base_lang = language.split("-")[0].lower() + return base_lang in cjk_languages + + def _process_word_timestamps_for_language( + self, words: List[str], starts: List[float] + ) -> List[tuple[str, float]]: + """Process word timestamps based on the current language. + + For CJK languages, Cartesia groups related characters in the same timestamp message. + For example, in Japanese a single message might be `['こ', 'ん', 'に', 'ち', 'は', '。']`. + We combine these into single words so the downstream aggregator can add natural + spacing between meaningful units rather than individual characters. + + For non-CJK languages, words are already properly separated and are used as-is. + + Args: + words: List of words/characters from Cartesia. + starts: List of start timestamps for each word/character. + + Returns: + List of (word, start_time) tuples processed for the language. + """ + current_language = self._settings.get("language", "en") + + # Check if this is a CJK language + if self._is_cjk_language(current_language): + # For CJK languages, combine all characters in this message into one word + # using the first character's start time + if words and starts: + combined_word = "".join(words) + first_start = starts[0] + return [(combined_word, first_start)] + else: + return [] + else: + # For non-CJK languages, use as-is + return list(zip(words, starts)) + def _build_msg( self, text: str = "", continue_transcript: bool = True, add_timestamps: bool = True ): @@ -351,9 +399,11 @@ class CartesiaTTSService(AudioContextWordTTSService): await self.add_word_timestamps([("TTSStoppedFrame", 0), ("Reset", 0)]) await self.remove_audio_context(msg["context_id"]) elif msg["type"] == "timestamps": - await self.add_word_timestamps( - list(zip(msg["word_timestamps"]["words"], msg["word_timestamps"]["start"])) + # Process the timestamps based on language before adding them + processed_timestamps = self._process_word_timestamps_for_language( + msg["word_timestamps"]["words"], msg["word_timestamps"]["start"] ) + await self.add_word_timestamps(processed_timestamps) elif msg["type"] == "chunk": await self.stop_ttfb_metrics() self.start_word_timestamps()