From 064b68aa014b4745d973f6afa62aa0c9291db91c Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 12 May 2026 18:59:14 -0400 Subject: [PATCH] Fix Cartesia CJK timestamp spacing --- changelog/4475.fixed.2.md | 1 + changelog/4475.fixed.md | 1 + src/pipecat/services/cartesia/tts.py | 35 ++++++--- tests/test_cartesia_tts.py | 111 +++++++++++++++++++++++++++ 4 files changed, 136 insertions(+), 12 deletions(-) create mode 100644 changelog/4475.fixed.2.md create mode 100644 changelog/4475.fixed.md create mode 100644 tests/test_cartesia_tts.py diff --git a/changelog/4475.fixed.2.md b/changelog/4475.fixed.2.md new file mode 100644 index 000000000..378fc5aef --- /dev/null +++ b/changelog/4475.fixed.2.md @@ -0,0 +1 @@ +- Fixed Cartesia TTS Chinese and Japanese timestamp grouping to preserve provider text spacing, avoiding artificial spaces when timestamp groups are reassembled downstream. diff --git a/changelog/4475.fixed.md b/changelog/4475.fixed.md new file mode 100644 index 000000000..3958544a2 --- /dev/null +++ b/changelog/4475.fixed.md @@ -0,0 +1 @@ +- Fixed Cartesia TTS Korean word timestamps to use normal spacing rules, preserving word boundaries and per-word timestamp alignment during downstream aggregation. diff --git a/src/pipecat/services/cartesia/tts.py b/src/pipecat/services/cartesia/tts.py index 5a2aa04f7..1080056ad 100644 --- a/src/pipecat/services/cartesia/tts.py +++ b/src/pipecat/services/cartesia/tts.py @@ -419,30 +419,30 @@ class CartesiaTTSService(WebsocketTTSService): """Convenience method to create a speed tag.""" return f'' - def _is_cjk_language(self, language: str) -> bool: - """Check if the given language is CJK (Chinese, Japanese, Korean). + def _is_chinese_or_japanese_language(self, language: str) -> bool: + """Check if the given language is Chinese or Japanese. Args: language: The language code to check. Returns: - True if the language is Chinese, Japanese, or Korean. + True if the language is Chinese or Japanese. """ - cjk_languages = {"zh", "ja", "ko"} base_lang = language.split("-")[0].lower() - return base_lang in cjk_languages + return base_lang in {"zh", "ja"} def _process_word_timestamps_for_language( self, words: list[str], starts: list[float] ) -> list[tuple[str, float]]: """Process word timestamps based on the current language. - For CJK languages, Cartesia groups related characters in the same timestamp message. + For Chinese and Japanese, Cartesia groups related characters in the same timestamp + message. For example, in Japanese a single message might be `['こ', 'ん', 'に', 'ち', 'は', '。']`. We combine these into single words so the downstream aggregator can add natural spacing between meaningful units rather than individual characters. - For non-CJK languages, words are already properly separated and are used as-is. + For other languages, words are already properly separated and are used as-is. Args: words: List of words/characters from Cartesia. @@ -453,10 +453,10 @@ class CartesiaTTSService(WebsocketTTSService): """ current_language = assert_given(self._settings.language) - # Check if this is a CJK language (if language is None, treat as non-CJK) - if current_language and self._is_cjk_language(current_language): - # For CJK languages, combine all characters in this message into one word - # using the first character's start time + # Check if this is a Chinese/Japanese language (if language is None, treat as other) + if current_language and self._is_chinese_or_japanese_language(current_language): + # For Chinese/Japanese, combine all characters in this message into one word + # using the first character's start time. if words and starts: combined_word = "".join(words) first_start = starts[0] @@ -467,6 +467,11 @@ class CartesiaTTSService(WebsocketTTSService): # For non-CJK languages, use as-is return list(zip(words, starts)) + def _word_timestamps_include_inter_frame_spaces(self) -> bool: + """Whether timestamp text should be treated as carrying its own spacing.""" + current_language = assert_given(self._settings.language) + return bool(current_language and self._is_chinese_or_japanese_language(current_language)) + def _build_msg( self, text: str = "", @@ -660,7 +665,13 @@ class CartesiaTTSService(WebsocketTTSService): processed_timestamps = self._process_word_timestamps_for_language( msg["word_timestamps"]["words"], msg["word_timestamps"]["start"] ) - await self.add_word_timestamps(processed_timestamps, ctx_id) + await self.add_word_timestamps( + processed_timestamps, + ctx_id, + includes_inter_frame_spaces=( + True if self._word_timestamps_include_inter_frame_spaces() else None + ), + ) elif msg["type"] == "chunk": frame = TTSAudioRawFrame( audio=base64.b64decode(msg["data"]), diff --git a/tests/test_cartesia_tts.py b/tests/test_cartesia_tts.py new file mode 100644 index 000000000..c14dc7409 --- /dev/null +++ b/tests/test_cartesia_tts.py @@ -0,0 +1,111 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.settings import TTSSettings +from pipecat.utils.string import TextPartForConcatenation, concatenate_aggregated_text + + +def _service(language: str) -> CartesiaTTSService: + service = CartesiaTTSService.__new__(CartesiaTTSService) + service._settings = TTSSettings(language=language) + return service + + +def _process_word_timestamps( + words: list[str], starts: list[float], language: str +) -> list[tuple[str, float]]: + return _service(language)._process_word_timestamps_for_language(words, starts) + + +def _concatenate_processed_timestamps( + timestamp_groups: list[tuple[list[str], list[float]]], language: str +) -> str: + service = _service(language) + text_parts = [] + for words, starts in timestamp_groups: + processed_timestamps = service._process_word_timestamps_for_language(words, starts) + includes_inter_frame_spaces = service._word_timestamps_include_inter_frame_spaces() + text_parts.extend( + TextPartForConcatenation( + word, + includes_inter_part_spaces=includes_inter_frame_spaces, + ) + for word, _timestamp in processed_timestamps + ) + return concatenate_aggregated_text(text_parts) + + +def test_cartesia_chinese_word_timestamps_join_without_spaces(): + assert _process_word_timestamps( + words=["你", "好", "。"], + starts=[0.0, 0.1, 0.2], + language="zh", + ) == [("你好。", 0.0)] + + +def test_cartesia_japanese_word_timestamps_join_without_spaces(): + assert _process_word_timestamps( + words=["こ", "ん", "に", "ち", "は", "。"], + starts=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5], + language="ja", + ) == [("こんにちは。", 0.0)] + + +def test_cartesia_korean_word_timestamps_preserve_words_and_timestamps(): + assert _process_word_timestamps( + words=["안녕하세요", "반갑습니다"], + starts=[0.0, 0.2], + language="ko", + ) == [("안녕하세요", 0.0), ("반갑습니다", 0.2)] + + +def test_cartesia_korean_word_timestamps_do_not_join_latin_and_hangul(): + assert _process_word_timestamps( + words=["AI", "어시스턴트입니다."], + starts=[3.7026982, 4.1999383], + language="ko", + ) == [("AI", 3.7026982), ("어시스턴트입니다.", 4.1999383)] + + +def test_cartesia_japanese_timestamp_groups_reassemble_without_spaces(): + assert ( + _concatenate_processed_timestamps( + [ + (["こ", "ん", "に", "ち", "は", "、", "私"], [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7]), + (["は", "あ", "な", "た", "の"], [1.0, 1.1, 1.2, 1.3, 1.4]), + ], + language="ja", + ) + == "こんにちは、私はあなたの" + ) + + +def test_cartesia_chinese_timestamp_groups_reassemble_without_spaces(): + assert ( + _concatenate_processed_timestamps( + [ + (["你", "好", "，", "我", "是"], [0.1, 0.2, 0.3, 0.4, 0.5]), + (["你", "的", "智", "能"], [1.0, 1.1, 1.2, 1.3]), + ], + language="zh", + ) + == "你好，我是你的智能" + ) + + +def test_cartesia_korean_timestamp_groups_reassemble_with_spaces(): + assert ( + _concatenate_processed_timestamps( + [ + (["저는"], [1.6]), + (["여러분의"], [1.8]), + (["AI", "어시스턴트입니다."], [3.7, 4.2]), + ], + language="ko", + ) + == "저는 여러분의 AI 어시스턴트입니다." + )