Merge pull request #2840 from Rickaym/fix--excess-space-in-elevelabs-word-timestamp-joins

fix: handle ElevenLabs partial word concatenation across alignment chunks gracefully

src/pipecat/services/elevenlabs/tts.py

@@ -168,16 +168,24 @@ def build_elevenlabs_voice_settings(
 
 
 def calculate_word_times(
-    alignment_info: Mapping[str, Any], cumulative_time: float
+    alignment_info: Mapping[str, Any],
-) -> List[Tuple[str, float]]:
+    cumulative_time: float,
+    partial_word: str = "",
+    partial_word_start_time: float = 0.0,
+) -> tuple[List[Tuple[str, float]], str, float]:
     """Calculate word timestamps from character alignment information.
 
     Args:
         alignment_info: Character alignment data from ElevenLabs API.
         cumulative_time: Base time offset for this chunk.
+        partial_word: Partial word carried over from previous chunk.
+        partial_word_start_time: Start time of the partial word.
 
     Returns:
-        List of (word, timestamp) tuples.
+        Tuple of (word_times, new_partial_word, new_partial_word_start_time):
+        - word_times: List of (word, timestamp) tuples for complete words
+        - new_partial_word: Incomplete word at end of chunk (empty if chunk ends with space)
+        - new_partial_word_start_time: Start time of the incomplete word
     """
     chars = alignment_info["chars"]
     char_start_times_ms = alignment_info["charStartTimesMs"]
@@ -186,41 +194,37 @@ def calculate_word_times(
         logger.error(
             f"calculate_word_times: length mismatch - chars={len(chars)}, times={len(char_start_times_ms)}"
         )
-        return []
+        return ([], partial_word, partial_word_start_time)
 
     # Build words and track their start positions
     words = []
-    word_start_indices = []
+    word_start_times = []
-    current_word = ""
+    current_word = partial_word  # Start with any partial word from previous chunk
-    word_start_index = None
+    word_start_time = partial_word_start_time if partial_word else None
 
     for i, char in enumerate(chars):
         if char == " ":
             # End of current word
             if current_word:  # Only add non-empty words
                 words.append(current_word)
-                word_start_indices.append(word_start_index)
+                word_start_times.append(word_start_time)
                 current_word = ""
-                word_start_index = None
+                word_start_time = None
         else:
             # Building a word
-            if word_start_index is None:  # First character of new word
+            if word_start_time is None:  # First character of new word
-                word_start_index = i
+                # Convert from milliseconds to seconds and add cumulative offset
+                word_start_time = cumulative_time + (char_start_times_ms[i] / 1000.0)
             current_word += char
 
-    # Handle the last word if there's no trailing space
+    # Build result for complete words
-    if current_word and word_start_index is not None:
+    word_times = list(zip(words, word_start_times))
-        words.append(current_word)
-        word_start_indices.append(word_start_index)
 
-    # Calculate timestamps for each word
+    # Return any incomplete word at the end of this chunk
-    word_times = []
+    new_partial_word = current_word if current_word else ""
-    for word, start_idx in zip(words, word_start_indices):
+    new_partial_word_start_time = word_start_time if word_start_time is not None else 0.0
-        # Convert from milliseconds to seconds and add cumulative offset
-        start_time_seconds = cumulative_time + (char_start_times_ms[start_idx] / 1000.0)
-        word_times.append((word, start_time_seconds))
 
-    return word_times
+    return (word_times, new_partial_word, new_partial_word_start_time)
 
 
 class ElevenLabsTTSService(AudioContextWordTTSService):
@@ -332,6 +336,9 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
         # there's an interruption or TTSStoppedFrame.
         self._started = False
         self._cumulative_time = 0
+        # Track partial words that span across alignment chunks
+        self._partial_word = ""
+        self._partial_word_start_time = 0.0
 
         # Context management for v1 multi API
         self._context_id = None
@@ -570,6 +577,8 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
                 logger.error(f"Error closing context on interruption: {e}")
             self._context_id = None
             self._started = False
+            self._partial_word = ""
+            self._partial_word_start_time = 0.0
 
     async def _receive_messages(self):
         """Handle incoming WebSocket messages from ElevenLabs."""
@@ -609,7 +618,14 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
 
             if msg.get("alignment"):
                 alignment = msg["alignment"]
-                word_times = calculate_word_times(alignment, self._cumulative_time)
+                word_times, self._partial_word, self._partial_word_start_time = (
+                    calculate_word_times(
+                        alignment,
+                        self._cumulative_time,
+                        self._partial_word,
+                        self._partial_word_start_time,
+                    )
+                )
 
                 if word_times:
                     await self.add_word_timestamps(word_times)
@@ -683,6 +699,8 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
                     yield TTSStartedFrame()
                     self._started = True
                     self._cumulative_time = 0
+                    self._partial_word = ""
+                    self._partial_word_start_time = 0.0
                     # If a context ID does not exist, create a new one and
                     # register it. If an ID exists, that means the Pipeline is
                     # configured for allow_interruptions=False, so continue
@@ -809,6 +827,10 @@ class ElevenLabsHttpTTSService(WordTTSService):
         # Store previous text for context within a turn
         self._previous_text = ""
 
+        # Track partial words that span across alignment chunks
+        self._partial_word = ""
+        self._partial_word_start_time = 0.0
+
     def language_to_service_language(self, language: Language) -> Optional[str]:
         """Convert pipecat Language to ElevenLabs language code.
 
@@ -836,6 +858,8 @@ class ElevenLabsHttpTTSService(WordTTSService):
         self._cumulative_time = 0
         self._started = False
         self._previous_text = ""
+        self._partial_word = ""
+        self._partial_word_start_time = 0.0
         logger.debug(f"{self}: Reset internal state")
 
     async def start(self, frame: StartFrame):
@@ -870,11 +894,13 @@ class ElevenLabsHttpTTSService(WordTTSService):
     def calculate_word_times(self, alignment_info: Mapping[str, Any]) -> List[Tuple[str, float]]:
         """Calculate word timing from character alignment data.
 
+        This method handles partial words that may span across multiple alignment chunks.
+
         Args:
             alignment_info: Character timing data from ElevenLabs.
 
         Returns:
-            List of (word, timestamp) pairs.
+            List of (word, timestamp) pairs for complete words in this chunk.
 
         Example input data::
 
@@ -900,30 +926,28 @@ class ElevenLabsHttpTTSService(WordTTSService):
         # Build the words and find their start times
         words = []
         word_start_times = []
-        current_word = ""
+        # Start with any partial word from previous chunk
-        first_char_idx = -1
+        current_word = self._partial_word
+        word_start_time = self._partial_word_start_time if self._partial_word else None
 
         for i, char in enumerate(chars):
             if char == " ":
                 if current_word:  # Only add non-empty words
                     words.append(current_word)
-                    # Use time of the first character of the word, offset by cumulative time
+                    word_start_times.append(word_start_time)
-                    word_start_times.append(
-                        self._cumulative_time + char_start_times[first_char_idx]
-                    )
                     current_word = ""
-                    first_char_idx = -1
+                    word_start_time = None
             else:
-                if not current_word:  # This is the first character of a new word
+                if word_start_time is None:  # First character of a new word
-                    first_char_idx = i
+                    # Use time of the first character of the word, offset by cumulative time
+                    word_start_time = self._cumulative_time + char_start_times[i]
                 current_word += char
 
-        # Don't forget the last word if there's no trailing space
+        # Store any incomplete word at the end of this chunk
-        if current_word and first_char_idx >= 0:
+        self._partial_word = current_word if current_word else ""
-            words.append(current_word)
+        self._partial_word_start_time = word_start_time if word_start_time is not None else 0.0
-            word_start_times.append(self._cumulative_time + char_start_times[first_char_idx])
 
-        # Create word-time pairs
+        # Create word-time pairs for complete words only
         word_times = list(zip(words, word_start_times))
 
         return word_times