wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[inworld] Allow Async delivery of timestamps info

Browse Source 
* speed up first audio chunk latency
This commit is contained in:
Ian Lee
2026-02-02 18:18:02 -08:00
parent 432870cc36
commit 94f01af545
2 changed files with 23 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
changelog/3625.added.md Normal file
View File

@@ -0,0 +1 @@
- Added `"timestampTransportStrategy": "ASYNC"` to `InworldAITTSService`. This allows timestamps info to trail audio chunks arrival, resulting in much better first audio chunk latency

30
src/pipecat/services/inworld/tts.py
View File

@@ -17,7 +17,7 @@ import asyncio
import base64 import base64
import json import json
import uuid import uuid
from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple from typing import Any, AsyncGenerator, Dict, List, Literal, Optional, Tuple
import aiohttp import aiohttp
import websockets import websockets
@@ -65,10 +65,12 @@ class InworldHttpTTSService(WordTTSService):
Parameters: Parameters:
temperature: Temperature for speech synthesis. temperature: Temperature for speech synthesis.
speaking_rate: Speaking rate for speech synthesis. speaking_rate: Speaking rate for speech synthesis.
timestamp_transport_strategy: The strategy to use for timestamp transport.
""" """
temperature: Optional[float] = None temperature: Optional[float] = None
speaking_rate: Optional[float] = None speaking_rate: Optional[float] = None
timestamp_transport_strategy: Optional[Literal["ASYNC", "SYNC"]] = None
def __init__( def __init__(
self, self,
@@ -128,6 +130,8 @@ class InworldHttpTTSService(WordTTSService):
self._settings["temperature"] = params.temperature self._settings["temperature"] = params.temperature
if params.speaking_rate is not None: if params.speaking_rate is not None:
self._settings["audioConfig"]["speakingRate"] = params.speaking_rate self._settings["audioConfig"]["speakingRate"] = params.speaking_rate
if params.timestamp_transport_strategy is not None:
self._settings["timestampTransportStrategy"] = params.timestamp_transport_strategy
self._cumulative_time = 0.0 self._cumulative_time = 0.0
@@ -240,6 +244,8 @@ class InworldHttpTTSService(WordTTSService):
# Use WORD timestamps for simplicity and correct spacing/capitalization # Use WORD timestamps for simplicity and correct spacing/capitalization
payload["timestampType"] = self._timestamp_type payload["timestampType"] = self._timestamp_type
if "timestampTransportStrategy" in self._settings:
payload["timestampTransportStrategy"] = self._settings["timestampTransportStrategy"]
request_id = str(uuid.uuid4()) request_id = str(uuid.uuid4())
headers = { headers = {
@@ -427,6 +433,7 @@ class InworldTTSService(AudioContextWordTTSService):
flushing of buffered text to achieve minimal latency while flushing of buffered text to achieve minimal latency while
maintaining high quality audio output. If None (default), maintaining high quality audio output. If None (default),
automatically set based on aggregate_sentences. automatically set based on aggregate_sentences.
timestamp_transport_strategy: The strategy to use for timestamp transport.
""" """
temperature: Optional[float] = None temperature: Optional[float] = None
@@ -434,7 +441,8 @@ class InworldTTSService(AudioContextWordTTSService):
apply_text_normalization: Optional[str] = None apply_text_normalization: Optional[str] = None
max_buffer_delay_ms: Optional[int] = None max_buffer_delay_ms: Optional[int] = None
buffer_char_threshold: Optional[int] = None buffer_char_threshold: Optional[int] = None
auto_mode: Optional[bool] = None auto_mode: Optional[bool] = True
timestamp_transport_strategy: Optional[Literal["ASYNC", "SYNC"]] = None
def __init__( def __init__(
self, self,
@@ -494,6 +502,8 @@ class InworldTTSService(AudioContextWordTTSService):
self._settings["audioConfig"]["speakingRate"] = params.speaking_rate self._settings["audioConfig"]["speakingRate"] = params.speaking_rate
if params.apply_text_normalization is not None: if params.apply_text_normalization is not None:
self._settings["applyTextNormalization"] = params.apply_text_normalization self._settings["applyTextNormalization"] = params.apply_text_normalization
if params.timestamp_transport_strategy is not None:
self._settings["timestampTransportStrategy"] = params.timestamp_transport_strategy
if params.auto_mode is not None: if params.auto_mode is not None:
self._settings["autoMode"] = params.auto_mode self._settings["autoMode"] = params.auto_mode
@@ -815,12 +825,12 @@ class InworldTTSService(AudioContextWordTTSService):
if ctx_id: if ctx_id:
await self.append_to_audio_context(ctx_id, frame) await self.append_to_audio_context(ctx_id, frame)
# timestampInfo is inside audioChunk # timestampInfo is inside audioChunk
timestamp_info = audio_chunk.get("timestampInfo") timestamp_info = audio_chunk.get("timestampInfo")
if timestamp_info: if timestamp_info:
word_times = self._calculate_word_times(timestamp_info) word_times = self._calculate_word_times(timestamp_info)
if word_times: if word_times:
await self.add_word_timestamps(word_times, ctx_id) await self.add_word_timestamps(word_times, ctx_id)
# Handle context created confirmation # Handle context created confirmation
if "contextCreated" in result: if "contextCreated" in result:
@@ -884,6 +894,10 @@ class InworldTTSService(AudioContextWordTTSService):
create_config["applyTextNormalization"] = self._settings["applyTextNormalization"] create_config["applyTextNormalization"] = self._settings["applyTextNormalization"]
if "autoMode" in self._settings: if "autoMode" in self._settings:
create_config["autoMode"] = self._settings["autoMode"] create_config["autoMode"] = self._settings["autoMode"]
if "timestampTransportStrategy" in self._settings:
create_config["timestampTransportStrategy"] = self._settings[
"timestampTransportStrategy"
]
# Set buffer settings for timely audio generation. # Set buffer settings for timely audio generation.
# Use provided values or defaults that work well for streaming LLM output. # Use provided values or defaults that work well for streaming LLM output.