Merge pull request #4344 from pipecat-ai/mb/11labs-normalized-alignment

changelog/4344.fixed.md

@@ -0,0 +1 @@
+- Fixed `ElevenLabsTTSService` and `ElevenLabsHttpTTSService` emitting word timestamps and `TTSTextFrame` content that matched the input text instead of the spoken audio when a pronunciation dictionary (`pronunciation_dictionary_locators`) or text normalization rewrote the input. Both services now consume ElevenLabs' normalized alignment, so downstream consumers (captions, transcripts, context aggregation) reflect what the listener actually hears.

src/pipecat/services/elevenlabs/tts.py

@@ -245,6 +245,35 @@ class ElevenLabsHttpTTSSettings(TTSSettings):
     )
 
 
+def _strip_leading_space(
+    alignment: Mapping[str, Any], keys: tuple[str, str, str]
+) -> Mapping[str, Any]:
+    """Return alignment with a prepended space char removed, if present.
+
+    Normalized alignment chunks from ElevenLabs begin with a leading space that
+    marks the prosody/chunk boundary. Left in place, it would prematurely
+    terminate a partial word carried over from the previous chunk. Stripping it
+    is lossless for timing: the dropped space's duration is still reflected in
+    the next char's `charStartTimesMs`, and the chunk's last-element values
+    (used to advance cumulative time) are untouched.
+
+    Args:
+        alignment: Alignment dict from the API.
+        keys: Tuple of (chars_key, start_times_key, durations_or_end_times_key)
+            naming the three parallel arrays — these differ between the
+            WebSocket and HTTP response schemas.
+    """
+    chars_key, starts_key, tail_key = keys
+    chars = alignment.get(chars_key) or []
+    if chars and chars[0] == " ":
+        return {
+            chars_key: chars[1:],
+            starts_key: alignment.get(starts_key, [])[1:],
+            tail_key: alignment.get(tail_key, [])[1:],
+        }
+    return alignment
+
+
 def calculate_word_times(
     alignment_info: Mapping[str, Any],
     cumulative_time: float,
@@ -790,8 +819,15 @@ class ElevenLabsTTSService(WebsocketTTSService):
                 frame = TTSAudioRawFrame(audio, self.sample_rate, 1, context_id=received_ctx_id)
                 await self.append_to_audio_context(received_ctx_id, frame)
 
-            if msg.get("alignment"):
-                alignment = msg["alignment"]
+            if msg.get("normalizedAlignment"):
+                # Use normalizedAlignment (what was actually spoken) rather than
+                # alignment (the input text), so word timestamps stay accurate
+                # when a pronunciation dictionary or text normalization rewrites
+                # the input.
+                alignment = _strip_leading_space(
+                    msg["normalizedAlignment"],
+                    ("chars", "charStartTimesMs", "charDurationsMs"),
+                )
                 word_times, self._partial_word, self._partial_word_start_time = (
                     calculate_word_times(
                         alignment,
@@ -1296,21 +1332,30 @@ class ElevenLabsHttpTTSService(TTSService):
                                 audio, self.sample_rate, 1, context_id=context_id
                             )
 
-                        # Process alignment if present
-                        if data and "alignment" in data:
-                            alignment = data["alignment"]
-                            if alignment:  # Ensure alignment is not None
-                                # Get end time of the last character in this chunk
-                                char_end_times = alignment.get("character_end_times_seconds", [])
-                                if char_end_times:
-                                    chunk_end_time = char_end_times[-1]
-                                    # Update to the longest end time seen so far
-                                    utterance_duration = max(utterance_duration, chunk_end_time)
+                        # Process alignment if present. Use normalized_alignment
+                        # (what was actually spoken) so word timestamps stay
+                        # accurate when a pronunciation dictionary or text
+                        # normalization rewrites the input.
+                        if data and data.get("normalized_alignment"):
+                            alignment = _strip_leading_space(
+                                data["normalized_alignment"],
+                                (
+                                    "characters",
+                                    "character_start_times_seconds",
+                                    "character_end_times_seconds",
+                                ),
+                            )
+                            # Get end time of the last character in this chunk
+                            char_end_times = alignment.get("character_end_times_seconds", [])
+                            if char_end_times:
+                                chunk_end_time = char_end_times[-1]
+                                # Update to the longest end time seen so far
+                                utterance_duration = max(utterance_duration, chunk_end_time)
 
-                                # Calculate word timestamps
-                                word_times = self.calculate_word_times(alignment)
-                                if word_times:
-                                    await self.add_word_timestamps(word_times, context_id)
+                            # Calculate word timestamps
+                            word_times = self.calculate_word_times(alignment)
+                            if word_times:
+                                await self.add_word_timestamps(word_times, context_id)
                     except json.JSONDecodeError as e:
                         logger.warning(f"Failed to parse JSON from stream: {e}")
                         continue