Merge pull request #2304 from pipecat-ai/mb/cartesia-cjk-lang-support

CartesiaTTSService: Add CJK lang support for word timestamps

CHANGELOG.md

@@ -32,6 +32,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   average response time between user stop and bot start, as well as minimum and
   maximum response latency.
 
+- Added Chinese, Japanese, Korean word timestamp support to
+  `CartesiaTTSService`.
+
 ### Changed
 
 - Changed the default model for both `ElevenLabsTTSService` and

src/pipecat/services/cartesia/tts.py

@@ -216,6 +216,54 @@ class CartesiaTTSService(AudioContextWordTTSService):
         """
         return language_to_cartesia_language(language)
 
+    def _is_cjk_language(self, language: str) -> bool:
+        """Check if the given language is CJK (Chinese, Japanese, Korean).
+
+        Args:
+            language: The language code to check.
+
+        Returns:
+            True if the language is Chinese, Japanese, or Korean.
+        """
+        cjk_languages = {"zh", "ja", "ko"}
+        base_lang = language.split("-")[0].lower()
+        return base_lang in cjk_languages
+
+    def _process_word_timestamps_for_language(
+        self, words: List[str], starts: List[float]
+    ) -> List[tuple[str, float]]:
+        """Process word timestamps based on the current language.
+
+        For CJK languages, Cartesia groups related characters in the same timestamp message.
+        For example, in Japanese a single message might be `['こ', 'ん', 'に', 'ち', 'は', '。']`.
+        We combine these into single words so the downstream aggregator can add natural
+        spacing between meaningful units rather than individual characters.
+
+        For non-CJK languages, words are already properly separated and are used as-is.
+
+        Args:
+            words: List of words/characters from Cartesia.
+            starts: List of start timestamps for each word/character.
+
+        Returns:
+            List of (word, start_time) tuples processed for the language.
+        """
+        current_language = self._settings.get("language", "en")
+
+        # Check if this is a CJK language
+        if self._is_cjk_language(current_language):
+            # For CJK languages, combine all characters in this message into one word
+            # using the first character's start time
+            if words and starts:
+                combined_word = "".join(words)
+                first_start = starts[0]
+                return [(combined_word, first_start)]
+            else:
+                return []
+        else:
+            # For non-CJK languages, use as-is
+            return list(zip(words, starts))
+
     def _build_msg(
         self, text: str = "", continue_transcript: bool = True, add_timestamps: bool = True
     ):
@@ -351,9 +399,11 @@ class CartesiaTTSService(AudioContextWordTTSService):
                 await self.add_word_timestamps([("TTSStoppedFrame", 0), ("Reset", 0)])
                 await self.remove_audio_context(msg["context_id"])
             elif msg["type"] == "timestamps":
-                await self.add_word_timestamps(
-                    list(zip(msg["word_timestamps"]["words"], msg["word_timestamps"]["start"]))
+                # Process the timestamps based on language before adding them
+                processed_timestamps = self._process_word_timestamps_for_language(
+                    msg["word_timestamps"]["words"], msg["word_timestamps"]["start"]
                 )
+                await self.add_word_timestamps(processed_timestamps)
             elif msg["type"] == "chunk":
                 await self.stop_ttfb_metrics()
                 self.start_word_timestamps()