Merge pull request #4424 from pipecat-ai/mb/revert-elevenlabs-tts-alignment

fix(elevenlabs): only use normalizedAlignment when pronunciation dict is set

changelog/4424.fixed.md

@@ -0,0 +1 @@
+- Fixed `ElevenLabsTTSService` and `ElevenLabsHttpTTSService` writing romanized/normalized text to the LLM context. With non-Latin input (e.g., Chinese), the assistant transcript was getting populated with pinyin (`Ni Hao !` instead of `你好！`), which then degraded subsequent LLM turns. The services now consume `alignment` by default and only switch to `normalizedAlignment` / `normalized_alignment` when `pronunciation_dictionary_locators` is configured (where `alignment` has overlapping restarts that produce duplicated/garbled words, per #4316). Both fields are read with preferred-with-fallback semantics since each is nullable per the API schema.

src/pipecat/services/elevenlabs/tts.py

@@ -248,17 +248,56 @@ class ElevenLabsHttpTTSSettings(TTSSettings):
     )
 
 
+def _select_alignment(
+    msg: Mapping[str, Any],
+    *,
+    normalized_key: str,
+    alignment_key: str,
+    prefer_normalized: bool,
+) -> Mapping[str, Any] | None:
+    """Pick the alignment field to use from a TTS message, with fallback.
+
+    ElevenLabs returns two alignment fields per chunk:
+
+    - ``normalized_key`` (``normalizedAlignment`` for WebSocket,
+      ``normalized_alignment`` for HTTP): the post-normalized form of what was
+      spoken - pronunciation-dictionary substitutions, text normalization, or
+      romanization of non-Latin scripts (e.g., Chinese rendered as pinyin).
+    - ``alignment_key`` (``alignment``): the original input characters.
+
+    Prefer ``normalized`` only when a pronunciation dictionary is configured -
+    that's the case where ``alignment`` has overlapping restarts that produce
+    duplicated/garbled words (issue #4316). Otherwise prefer ``alignment`` so
+    the LLM context preserves the original input rather than the normalized
+    form. Fall back to the other field if the preferred one is missing or
+    null - the API schema marks both as nullable.
+
+    Args:
+        msg: TTS response message from ElevenLabs.
+        normalized_key: Key for the normalized-alignment field on this transport.
+        alignment_key: Key for the original-alignment field on this transport.
+        prefer_normalized: True iff the caller is using pronunciation dictionaries.
+
+    Returns:
+        The chosen alignment dict, or ``None`` if both fields are absent/null.
+    """
+    if prefer_normalized:
+        return msg.get(normalized_key) or msg.get(alignment_key)
+    return msg.get(alignment_key) or msg.get(normalized_key)
+
+
 def _strip_utterance_leading_spaces(
     alignment: Mapping[str, Any], keys: tuple[str, str, str], should_strip: bool
 ) -> Mapping[str, Any]:
     """Return alignment with utterance-leading space chars removed, if requested.
 
-    Normalized alignment chunks from ElevenLabs often begin with a space. On the
-    first chunk of an utterance, that space is leading whitespace and should not
-    become a text token. On subsequent chunks, however, a leading space can be a
-    real inter-word separator (Flash models commonly split sentences this way),
-    so it must be preserved for ``calculate_word_times`` to flush any partial
-    word carried over from the previous chunk.
+    ElevenLabs Flash normalized alignment chunks can begin with a leading space
+    at the start of an utterance. Strip only utterance-leading spaces so bot
+    turn text does not start with whitespace. On subsequent chunks, however, a
+    leading space can be a real inter-word separator (Flash models commonly
+    split sentences this way), so it must be preserved for
+    ``calculate_word_times`` to flush any partial word carried over from the
+    previous chunk.
 
     Args:
         alignment: Alignment dict from the API.
@@ -829,13 +868,15 @@ class ElevenLabsTTSService(WebsocketTTSService):
                 frame = TTSAudioRawFrame(audio, self.sample_rate, 1, context_id=received_ctx_id)
                 await self.append_to_audio_context(received_ctx_id, frame)
 
-            if msg.get("normalizedAlignment"):
-                # Use normalizedAlignment (what was actually spoken) rather than
-                # alignment (the input text), so word timestamps stay accurate
-                # when a pronunciation dictionary or text normalization rewrites
-                # the input.
+            raw_alignment = _select_alignment(
+                msg,
+                normalized_key="normalizedAlignment",
+                alignment_key="alignment",
+                prefer_normalized=bool(self._pronunciation_dictionary_locators),
+            )
+            if raw_alignment:
                 alignment = _strip_utterance_leading_spaces(
-                    msg["normalizedAlignment"],
+                    raw_alignment,
                     ("chars", "charStartTimesMs", "charDurationsMs"),
                     received_ctx_id not in self._alignment_started_context_ids,
                 )
@@ -1353,13 +1394,15 @@ class ElevenLabsHttpTTSService(TTSService):
                                 audio, self.sample_rate, 1, context_id=context_id
                             )
 
-                        # Process alignment if present. Use normalized_alignment
-                        # (what was actually spoken) so word timestamps stay
-                        # accurate when a pronunciation dictionary or text
-                        # normalization rewrites the input.
-                        if data and data.get("normalized_alignment"):
+                        raw_alignment = data and _select_alignment(
+                            data,
+                            normalized_key="normalized_alignment",
+                            alignment_key="alignment",
+                            prefer_normalized=bool(self._pronunciation_dictionary_locators),
+                        )
+                        if raw_alignment:
                             alignment = _strip_utterance_leading_spaces(
-                                data["normalized_alignment"],
+                                raw_alignment,
                                 (
                                     "characters",
                                     "character_start_times_seconds",

tests/test_elevenlabs_tts.py

@@ -9,6 +9,7 @@
 from typing import Any
 
 from pipecat.services.elevenlabs.tts import (
+    _select_alignment,
     _strip_utterance_leading_spaces,
     calculate_word_times,
 )
@@ -90,3 +91,112 @@ def test_elevenlabs_alignment_strips_only_utterance_leading_spaces():
 
     assert first["chars"] == list("Hello")
     assert subsequent["chars"] == list(" world")
+
+
+def test_select_alignment_default_prefers_alignment():
+    msg = {
+        "alignment": _chunk("Hello"),
+        "normalizedAlignment": _chunk(" Hello"),
+    }
+    selected = _select_alignment(
+        msg,
+        normalized_key="normalizedAlignment",
+        alignment_key="alignment",
+        prefer_normalized=False,
+    )
+    assert selected is not None
+    assert selected["chars"] == list("Hello")
+
+
+def test_select_alignment_dictionary_mode_prefers_normalized():
+    msg = {
+        "alignment": _chunk("Hello"),
+        "normalizedAlignment": _chunk(" Hello"),
+    }
+    selected = _select_alignment(
+        msg,
+        normalized_key="normalizedAlignment",
+        alignment_key="alignment",
+        prefer_normalized=True,
+    )
+    assert selected is not None
+    assert selected["chars"] == list(" Hello")
+
+
+def test_select_alignment_falls_back_when_preferred_missing():
+    msg_default = {"normalizedAlignment": _chunk(" Hello")}
+    selected = _select_alignment(
+        msg_default,
+        normalized_key="normalizedAlignment",
+        alignment_key="alignment",
+        prefer_normalized=False,
+    )
+    assert selected is not None
+    assert selected["chars"] == list(" Hello")
+
+    msg_dict = {"alignment": _chunk("Hello")}
+    selected = _select_alignment(
+        msg_dict,
+        normalized_key="normalizedAlignment",
+        alignment_key="alignment",
+        prefer_normalized=True,
+    )
+    assert selected is not None
+    assert selected["chars"] == list("Hello")
+
+
+def test_select_alignment_falls_back_when_preferred_null():
+    msg = {"alignment": None, "normalizedAlignment": _chunk(" Hello")}
+    selected = _select_alignment(
+        msg,
+        normalized_key="normalizedAlignment",
+        alignment_key="alignment",
+        prefer_normalized=False,
+    )
+    assert selected is not None
+    assert selected["chars"] == list(" Hello")
+
+
+def test_select_alignment_returns_none_when_both_missing():
+    assert (
+        _select_alignment(
+            {},
+            normalized_key="normalizedAlignment",
+            alignment_key="alignment",
+            prefer_normalized=False,
+        )
+        is None
+    )
+    assert (
+        _select_alignment(
+            {"alignment": None, "normalizedAlignment": None},
+            normalized_key="normalizedAlignment",
+            alignment_key="alignment",
+            prefer_normalized=True,
+        )
+        is None
+    )
+
+
+def test_select_alignment_works_with_http_field_names():
+    msg = {
+        "alignment": {"characters": list("Hi")},
+        "normalized_alignment": {"characters": list(" Hi")},
+    }
+    selected = _select_alignment(
+        msg,
+        normalized_key="normalized_alignment",
+        alignment_key="alignment",
+        prefer_normalized=False,
+    )
+    assert selected is not None
+    assert selected["characters"] == list("Hi")
+
+    selected = _select_alignment(
+        msg,
+        normalized_key="normalized_alignment",
+        alignment_key="alignment",
+        prefer_normalized=True,
+    )
+    assert selected is not None
+    assert selected["characters"] == list(" Hi")