diff --git a/changelog/4344.fixed.md b/changelog/4344.fixed.md new file mode 100644 index 000000000..4e786f65c --- /dev/null +++ b/changelog/4344.fixed.md @@ -0,0 +1 @@ +- Fixed `ElevenLabsTTSService` and `ElevenLabsHttpTTSService` emitting word timestamps and `TTSTextFrame` content that matched the input text instead of the spoken audio when a pronunciation dictionary (`pronunciation_dictionary_locators`) or text normalization rewrote the input. Both services now consume ElevenLabs' normalized alignment, so downstream consumers (captions, transcripts, context aggregation) reflect what the listener actually hears. diff --git a/src/pipecat/services/elevenlabs/tts.py b/src/pipecat/services/elevenlabs/tts.py index 02e6383ff..f72864ef4 100644 --- a/src/pipecat/services/elevenlabs/tts.py +++ b/src/pipecat/services/elevenlabs/tts.py @@ -245,6 +245,35 @@ class ElevenLabsHttpTTSSettings(TTSSettings): ) +def _strip_leading_space( + alignment: Mapping[str, Any], keys: tuple[str, str, str] +) -> Mapping[str, Any]: + """Return alignment with a prepended space char removed, if present. + + Normalized alignment chunks from ElevenLabs begin with a leading space that + marks the prosody/chunk boundary. Left in place, it would prematurely + terminate a partial word carried over from the previous chunk. Stripping it + is lossless for timing: the dropped space's duration is still reflected in + the next char's `charStartTimesMs`, and the chunk's last-element values + (used to advance cumulative time) are untouched. + + Args: + alignment: Alignment dict from the API. + keys: Tuple of (chars_key, start_times_key, durations_or_end_times_key) + naming the three parallel arrays — these differ between the + WebSocket and HTTP response schemas. + """ + chars_key, starts_key, tail_key = keys + chars = alignment.get(chars_key) or [] + if chars and chars[0] == " ": + return { + chars_key: chars[1:], + starts_key: alignment.get(starts_key, [])[1:], + tail_key: alignment.get(tail_key, [])[1:], + } + return alignment + + def calculate_word_times( alignment_info: Mapping[str, Any], cumulative_time: float, @@ -790,8 +819,15 @@ class ElevenLabsTTSService(WebsocketTTSService): frame = TTSAudioRawFrame(audio, self.sample_rate, 1, context_id=received_ctx_id) await self.append_to_audio_context(received_ctx_id, frame) - if msg.get("alignment"): - alignment = msg["alignment"] + if msg.get("normalizedAlignment"): + # Use normalizedAlignment (what was actually spoken) rather than + # alignment (the input text), so word timestamps stay accurate + # when a pronunciation dictionary or text normalization rewrites + # the input. + alignment = _strip_leading_space( + msg["normalizedAlignment"], + ("chars", "charStartTimesMs", "charDurationsMs"), + ) word_times, self._partial_word, self._partial_word_start_time = ( calculate_word_times( alignment, @@ -1296,21 +1332,30 @@ class ElevenLabsHttpTTSService(TTSService): audio, self.sample_rate, 1, context_id=context_id ) - # Process alignment if present - if data and "alignment" in data: - alignment = data["alignment"] - if alignment: # Ensure alignment is not None - # Get end time of the last character in this chunk - char_end_times = alignment.get("character_end_times_seconds", []) - if char_end_times: - chunk_end_time = char_end_times[-1] - # Update to the longest end time seen so far - utterance_duration = max(utterance_duration, chunk_end_time) + # Process alignment if present. Use normalized_alignment + # (what was actually spoken) so word timestamps stay + # accurate when a pronunciation dictionary or text + # normalization rewrites the input. + if data and data.get("normalized_alignment"): + alignment = _strip_leading_space( + data["normalized_alignment"], + ( + "characters", + "character_start_times_seconds", + "character_end_times_seconds", + ), + ) + # Get end time of the last character in this chunk + char_end_times = alignment.get("character_end_times_seconds", []) + if char_end_times: + chunk_end_time = char_end_times[-1] + # Update to the longest end time seen so far + utterance_duration = max(utterance_duration, chunk_end_time) - # Calculate word timestamps - word_times = self.calculate_word_times(alignment) - if word_times: - await self.add_word_timestamps(word_times, context_id) + # Calculate word timestamps + word_times = self.calculate_word_times(alignment) + if word_times: + await self.add_word_timestamps(word_times, context_id) except json.JSONDecodeError as e: logger.warning(f"Failed to parse JSON from stream: {e}") continue