Merge pull request #4475 from pipecat-ai/mb/cartesia-korean-fix

Fix Cartesia CJK timestamp spacing

changelog/4475.fixed.2.md

@@ -0,0 +1 @@
+- Fixed Cartesia TTS Chinese and Japanese timestamp grouping to preserve provider text spacing, avoiding artificial spaces when timestamp groups are reassembled downstream.

changelog/4475.fixed.md

@@ -0,0 +1 @@
+- Fixed Cartesia TTS Korean word timestamps to use normal spacing rules, preserving word boundaries and per-word timestamp alignment during downstream aggregation.

src/pipecat/services/cartesia/tts.py

@@ -419,30 +419,30 @@ class CartesiaTTSService(WebsocketTTSService):
         """Convenience method to create a speed tag."""
         return f'<speed ratio="{speed}" />'
 
-    def _is_cjk_language(self, language: str) -> bool:
+    def _is_chinese_or_japanese_language(self, language: str) -> bool:
-        """Check if the given language is CJK (Chinese, Japanese, Korean).
+        """Check if the given language is Chinese or Japanese.
 
         Args:
             language: The language code to check.
 
         Returns:
-            True if the language is Chinese, Japanese, or Korean.
+            True if the language is Chinese or Japanese.
         """
-        cjk_languages = {"zh", "ja", "ko"}
         base_lang = language.split("-")[0].lower()
-        return base_lang in cjk_languages
+        return base_lang in {"zh", "ja"}
 
     def _process_word_timestamps_for_language(
         self, words: list[str], starts: list[float]
     ) -> list[tuple[str, float]]:
         """Process word timestamps based on the current language.
 
-        For CJK languages, Cartesia groups related characters in the same timestamp message.
+        For Chinese and Japanese, Cartesia groups related characters in the same timestamp
+        message.
         For example, in Japanese a single message might be `['こ', 'ん', 'に', 'ち', 'は', '。']`.
         We combine these into single words so the downstream aggregator can add natural
         spacing between meaningful units rather than individual characters.
 
-        For non-CJK languages, words are already properly separated and are used as-is.
+        For other languages, words are already properly separated and are used as-is.
 
         Args:
             words: List of words/characters from Cartesia.
@@ -453,10 +453,10 @@ class CartesiaTTSService(WebsocketTTSService):
         """
         current_language = assert_given(self._settings.language)
 
-        # Check if this is a CJK language (if language is None, treat as non-CJK)
+        # Check if this is a Chinese/Japanese language (if language is None, treat as other)
-        if current_language and self._is_cjk_language(current_language):
+        if current_language and self._is_chinese_or_japanese_language(current_language):
-            # For CJK languages, combine all characters in this message into one word
+            # For Chinese/Japanese, combine all characters in this message into one word
-            # using the first character's start time
+            # using the first character's start time.
             if words and starts:
                 combined_word = "".join(words)
                 first_start = starts[0]
@@ -467,6 +467,11 @@ class CartesiaTTSService(WebsocketTTSService):
             # For non-CJK languages, use as-is
             return list(zip(words, starts))
 
+    def _word_timestamps_include_inter_frame_spaces(self) -> bool:
+        """Whether timestamp text should be treated as carrying its own spacing."""
+        current_language = assert_given(self._settings.language)
+        return bool(current_language and self._is_chinese_or_japanese_language(current_language))
+
     def _build_msg(
         self,
         text: str = "",
@@ -660,7 +665,13 @@ class CartesiaTTSService(WebsocketTTSService):
                 processed_timestamps = self._process_word_timestamps_for_language(
                     msg["word_timestamps"]["words"], msg["word_timestamps"]["start"]
                 )
-                await self.add_word_timestamps(processed_timestamps, ctx_id)
+                await self.add_word_timestamps(
+                    processed_timestamps,
+                    ctx_id,
+                    includes_inter_frame_spaces=(
+                        True if self._word_timestamps_include_inter_frame_spaces() else None
+                    ),
+                )
             elif msg["type"] == "chunk":
                 frame = TTSAudioRawFrame(
                     audio=base64.b64decode(msg["data"]),

tests/test_cartesia_tts.py

@@ -0,0 +1,111 @@
+#
+# Copyright (c) 2024-2026, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+from pipecat.services.cartesia.tts import CartesiaTTSService
+from pipecat.services.settings import TTSSettings
+from pipecat.utils.string import TextPartForConcatenation, concatenate_aggregated_text
+
+
+def _service(language: str) -> CartesiaTTSService:
+    service = CartesiaTTSService.__new__(CartesiaTTSService)
+    service._settings = TTSSettings(language=language)
+    return service
+
+
+def _process_word_timestamps(
+    words: list[str], starts: list[float], language: str
+) -> list[tuple[str, float]]:
+    return _service(language)._process_word_timestamps_for_language(words, starts)
+
+
+def _concatenate_processed_timestamps(
+    timestamp_groups: list[tuple[list[str], list[float]]], language: str
+) -> str:
+    service = _service(language)
+    text_parts = []
+    for words, starts in timestamp_groups:
+        processed_timestamps = service._process_word_timestamps_for_language(words, starts)
+        includes_inter_frame_spaces = service._word_timestamps_include_inter_frame_spaces()
+        text_parts.extend(
+            TextPartForConcatenation(
+                word,
+                includes_inter_part_spaces=includes_inter_frame_spaces,
+            )
+            for word, _timestamp in processed_timestamps
+        )
+    return concatenate_aggregated_text(text_parts)
+
+
+def test_cartesia_chinese_word_timestamps_join_without_spaces():
+    assert _process_word_timestamps(
+        words=["你", "好", "。"],
+        starts=[0.0, 0.1, 0.2],
+        language="zh",
+    ) == [("你好。", 0.0)]
+
+
+def test_cartesia_japanese_word_timestamps_join_without_spaces():
+    assert _process_word_timestamps(
+        words=["こ", "ん", "に", "ち", "は", "。"],
+        starts=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5],
+        language="ja",
+    ) == [("こんにちは。", 0.0)]
+
+
+def test_cartesia_korean_word_timestamps_preserve_words_and_timestamps():
+    assert _process_word_timestamps(
+        words=["안녕하세요", "반갑습니다"],
+        starts=[0.0, 0.2],
+        language="ko",
+    ) == [("안녕하세요", 0.0), ("반갑습니다", 0.2)]
+
+
+def test_cartesia_korean_word_timestamps_do_not_join_latin_and_hangul():
+    assert _process_word_timestamps(
+        words=["AI", "어시스턴트입니다."],
+        starts=[3.7026982, 4.1999383],
+        language="ko",
+    ) == [("AI", 3.7026982), ("어시스턴트입니다.", 4.1999383)]
+
+
+def test_cartesia_japanese_timestamp_groups_reassemble_without_spaces():
+    assert (
+        _concatenate_processed_timestamps(
+            [
+                (["こ", "ん", "に", "ち", "は", "、", "私"], [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7]),
+                (["は", "あ", "な", "た", "の"], [1.0, 1.1, 1.2, 1.3, 1.4]),
+            ],
+            language="ja",
+        )
+        == "こんにちは、私はあなたの"
+    )
+
+
+def test_cartesia_chinese_timestamp_groups_reassemble_without_spaces():
+    assert (
+        _concatenate_processed_timestamps(
+            [
+                (["你", "好", "，", "我", "是"], [0.1, 0.2, 0.3, 0.4, 0.5]),
+                (["你", "的", "智", "能"], [1.0, 1.1, 1.2, 1.3]),
+            ],
+            language="zh",
+        )
+        == "你好，我是你的智能"
+    )
+
+
+def test_cartesia_korean_timestamp_groups_reassemble_with_spaces():
+    assert (
+        _concatenate_processed_timestamps(
+            [
+                (["저는"], [1.6]),
+                (["여러분의"], [1.8]),
+                (["AI", "어시스턴트입니다."], [3.7, 4.2]),
+            ],
+            language="ko",
+        )
+        == "저는 여러분의 AI 어시스턴트입니다."
+    )