From fef9e3ea32e55be42ac288488dc0152dc955d866 Mon Sep 17 00:00:00 2001 From: Ian Lee Date: Thu, 29 Jan 2026 11:42:14 -0800 Subject: [PATCH 1/5] Add auto_mode support for inworld plugin --- src/pipecat/services/inworld/tts.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/pipecat/services/inworld/tts.py b/src/pipecat/services/inworld/tts.py index 4e6ec3f4e..cb9078b30 100644 --- a/src/pipecat/services/inworld/tts.py +++ b/src/pipecat/services/inworld/tts.py @@ -414,6 +414,11 @@ class InworldTTSService(AudioContextWordTTSService): apply_text_normalization: Whether to apply text normalization. max_buffer_delay_ms: Maximum buffer delay in milliseconds. buffer_char_threshold: Buffer character threshold. + auto_mode: Whether to use auto mode. Recommended when texts are sent + in full sentences/phrases. When enabled, the server controls + flushing of buffered text to achieve minimal latency while + maintaining high quality audio output. If None (default), + automatically set based on aggregate_sentences. """ temperature: Optional[float] = None @@ -421,6 +426,7 @@ class InworldTTSService(AudioContextWordTTSService): apply_text_normalization: Optional[str] = None max_buffer_delay_ms: Optional[int] = None buffer_char_threshold: Optional[int] = None + auto_mode: Optional[bool] = None def __init__( self, @@ -432,6 +438,7 @@ class InworldTTSService(AudioContextWordTTSService): sample_rate: Optional[int] = None, encoding: str = "LINEAR16", params: InputParams = None, + aggregate_sentences: bool = True, **kwargs: Any, ): """Initialize the Inworld WebSocket TTS service. @@ -444,6 +451,7 @@ class InworldTTSService(AudioContextWordTTSService): sample_rate: Audio sample rate in Hz. encoding: Audio encoding format. params: Input parameters for Inworld WebSocket TTS configuration. + aggregate_sentences: Whether to aggregate sentences before synthesis. **kwargs: Additional arguments passed to the parent class. """ super().__init__( @@ -451,6 +459,7 @@ class InworldTTSService(AudioContextWordTTSService): push_stop_frames=True, pause_frame_processing=True, sample_rate=sample_rate, + aggregate_sentences=aggregate_sentences, **kwargs, ) @@ -475,6 +484,11 @@ class InworldTTSService(AudioContextWordTTSService): if params.apply_text_normalization is not None: self._settings["applyTextNormalization"] = params.apply_text_normalization + if params.auto_mode is not None: + self._settings["autoMode"] = params.auto_mode + else: + self._settings["autoMode"] = aggregate_sentences + self._buffer_settings = { "maxBufferDelayMs": params.max_buffer_delay_ms, "bufferCharThreshold": params.buffer_char_threshold, @@ -818,6 +832,8 @@ class InworldTTSService(AudioContextWordTTSService): create_config["temperature"] = self._settings["temperature"] if "applyTextNormalization" in self._settings: create_config["applyTextNormalization"] = self._settings["applyTextNormalization"] + if "autoMode" in self._settings: + create_config["autoMode"] = self._settings["autoMode"] # Set buffer settings for timely audio generation. # Use provided values or defaults that work well for streaming LLM output. From cbe131636df597fb70a927ce6a80ad194d6f8791 Mon Sep 17 00:00:00 2001 From: Ian Lee Date: Tue, 3 Feb 2026 11:41:33 -0800 Subject: [PATCH 2/5] add changelog --- 3593.added.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 3593.added.md diff --git a/3593.added.md b/3593.added.md new file mode 100644 index 000000000..432db3636 --- /dev/null +++ b/3593.added.md @@ -0,0 +1 @@ +- Added support for Inworld TTS Websocket Auto Mode for improved latency From d10467e043109e36ca603498ffe826ae6b9fd27e Mon Sep 17 00:00:00 2001 From: Ian Lee Date: Tue, 3 Feb 2026 21:55:53 -0800 Subject: [PATCH 3/5] update timestamps reset handling --- 3593.added.md | 1 - src/pipecat/services/inworld/tts.py | 47 ++++++++++++++++++++++++++--- 2 files changed, 43 insertions(+), 5 deletions(-) delete mode 100644 3593.added.md diff --git a/3593.added.md b/3593.added.md deleted file mode 100644 index 432db3636..000000000 --- a/3593.added.md +++ /dev/null @@ -1 +0,0 @@ -- Added support for Inworld TTS Websocket Auto Mode for improved latency diff --git a/src/pipecat/services/inworld/tts.py b/src/pipecat/services/inworld/tts.py index cb9078b30..16898c67f 100644 --- a/src/pipecat/services/inworld/tts.py +++ b/src/pipecat/services/inworld/tts.py @@ -499,6 +499,14 @@ class InworldTTSService(AudioContextWordTTSService): self._context_id = None self._started = False + # Track cumulative time across generations for monotonic timestamps within a turn. + # When auto_mode is enabled, the server controls generations and timestamps reset + # to 0 after each generation, as indicated by a "flushCompleted" message. We + # add _cumulative_time to maintain monotonically increasing timestamps. + self._cumulative_time = 0.0 + # Track the end time of the last word in the current generation + self._generation_end_time = 0.0 + self.set_voice(voice_id) self.set_model_name(model) @@ -559,27 +567,50 @@ class InworldTTSService(AudioContextWordTTSService): await super().push_frame(frame, direction) if isinstance(frame, (TTSStoppedFrame, InterruptionFrame)): self._started = False + logger.trace( + f"{self}: Resetting timestamp tracking due to {type(frame).__name__} - " + f"cumulative_time was {self._cumulative_time}" + ) + self._cumulative_time = 0.0 + self._generation_end_time = 0.0 if isinstance(frame, TTSStoppedFrame): await self.add_word_timestamps([("Reset", 0)]) def _calculate_word_times(self, timestamp_info: Dict[str, Any]) -> List[Tuple[str, float]]: """Calculate word timestamps from Inworld WebSocket API response. + Adds cumulative time offset to maintain monotonically increasing timestamps + across multiple generations within an agent turn. Also tracks the generation + end time for updating cumulative time on flush. + Args: timestamp_info: The timestamp information from Inworld API. Returns: - A list of (word, timestamp) tuples. + List of (word, timestamp) tuples with cumulative offset applied. """ word_times: List[Tuple[str, float]] = [] alignment = timestamp_info.get("wordAlignment", {}) words = alignment.get("words", []) start_times = alignment.get("wordStartTimeSeconds", []) + end_times = alignment.get("wordEndTimeSeconds", []) if words and start_times and len(words) == len(start_times): for i, word in enumerate(words): - word_times.append((word, start_times[i])) + word_start = self._cumulative_time + start_times[i] + word_times.append((word, word_start)) + + # Track cumulative end time for this generation + if end_times and len(end_times) > 0: + self._generation_end_time = self._cumulative_time + end_times[-1] + + logger.trace( + f"{self}: Word timestamps - raw_start_times={start_times}, " + f"cumulative_offset={self._cumulative_time}, " + f"adjusted_times={[t for _, t in word_times]}, " + f"generation_end_time={self._generation_end_time}" + ) return word_times @@ -604,6 +635,8 @@ class InworldTTSService(AudioContextWordTTSService): self._context_id = None self._started = False + self._cumulative_time = 0.0 + self._generation_end_time = 0.0 logger.trace(f"{self}: Interruption handled, context reset to None") def _get_websocket(self): @@ -693,6 +726,8 @@ class InworldTTSService(AudioContextWordTTSService): self._started = False self._context_id = None self._websocket = None + self._cumulative_time = 0.0 + self._generation_end_time = 0.0 await self._call_event_handler("on_disconnected") async def _receive_messages(self): @@ -779,9 +814,13 @@ class InworldTTSService(AudioContextWordTTSService): if "contextCreated" in result: logger.trace(f"{self}: Context created on server: {ctx_id}") - # Handle flush completion - context is still valid, just acknowledge it + # Handle flush completion, which indicates the end of a generation if "flushCompleted" in result: - logger.trace(f"{self}: Flush completed for context {ctx_id}") + logger.trace( + f"{self}: Generation completed - updating cumulative_time: " + f"{self._cumulative_time} -> {self._generation_end_time}" + ) + self._cumulative_time = self._generation_end_time # Handle context closed - context no longer exists on server if "contextClosed" in result: From 22398e1410f2b7b104a9b6eb11fd65d0cb8f569a Mon Sep 17 00:00:00 2001 From: Ian Lee Date: Thu, 5 Feb 2026 11:38:04 -0800 Subject: [PATCH 4/5] add changelog back --- changelog/3593.added.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelog/3593.added.md diff --git a/changelog/3593.added.md b/changelog/3593.added.md new file mode 100644 index 000000000..67b2394d2 --- /dev/null +++ b/changelog/3593.added.md @@ -0,0 +1,2 @@ +- Added support for Inworld TTS Websocket Auto Mode for improved latency +- Updated timestamps to be cumulative within an agent turn, using flushCompleted message as an indication of when timestamps from the server are reset to 0 \ No newline at end of file From 6eea40858e5e0f1cdfef6be7a8b1ce6e0a130e31 Mon Sep 17 00:00:00 2001 From: Ian Lee Date: Thu, 5 Feb 2026 15:10:36 -0800 Subject: [PATCH 5/5] fix lint and changelog --- changelog/3593.added.md | 1 - changelog/3593.change.md | 1 + src/pipecat/services/inworld/tts.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 changelog/3593.change.md diff --git a/changelog/3593.added.md b/changelog/3593.added.md index 67b2394d2..432db3636 100644 --- a/changelog/3593.added.md +++ b/changelog/3593.added.md @@ -1,2 +1 @@ - Added support for Inworld TTS Websocket Auto Mode for improved latency -- Updated timestamps to be cumulative within an agent turn, using flushCompleted message as an indication of when timestamps from the server are reset to 0 \ No newline at end of file diff --git a/changelog/3593.change.md b/changelog/3593.change.md new file mode 100644 index 000000000..a21549f36 --- /dev/null +++ b/changelog/3593.change.md @@ -0,0 +1 @@ +- Updated timestamps to be cumulative within an agent turn, using flushCompleted message as an indication of when timestamps from the server are reset to 0 diff --git a/src/pipecat/services/inworld/tts.py b/src/pipecat/services/inworld/tts.py index 16898c67f..fee6c49ac 100644 --- a/src/pipecat/services/inworld/tts.py +++ b/src/pipecat/services/inworld/tts.py @@ -501,7 +501,7 @@ class InworldTTSService(AudioContextWordTTSService): # Track cumulative time across generations for monotonic timestamps within a turn. # When auto_mode is enabled, the server controls generations and timestamps reset - # to 0 after each generation, as indicated by a "flushCompleted" message. We + # to 0 after each generation, as indicated by a "flushCompleted" message. We # add _cumulative_time to maintain monotonically increasing timestamps. self._cumulative_time = 0.0 # Track the end time of the last word in the current generation