wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(tts): add includes_inter_frame_spaces flag to word-timestamp API

Browse Source 
Some TTS providers (e.g. Inworld) return verbatim tokens where spaces and
punctuation are already embedded in the token text. When downstream consumers
join these tokens with an extra space they produce "hello , world" instead of
"hello, world".

Add an opt-in `includes_inter_frame_spaces: bool = False` parameter to
`add_word_timestamps` / `_add_word_timestamps`. The flag is threaded through
`_WordTimestampEntry` and stamped onto every emitted `TTSTextFrame`.
Defaults to `False` — no behaviour change for existing services.

`InworldTTSService` passes `includes_inter_frame_spaces=True` and stops
pre-processing tokens in `_calculate_word_times`, returning them verbatim.

Tests added to `test_tts_frame_ordering.py` covering both HTTP and WebSocket
delivery paths: verbatim text preservation, PTS ordering, text-before-audio
ordering, and the Inworld punctuation-token scenario.

Made-with: Cursor
This commit is contained in:
Ian Lee
2026-04-18 12:03:32 -07:00
parent fc1c3b48dc
commit b435ddfa44
4 changed files with 291 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/pipecat/services/inworld/tts.py
View File

@@ -410,7 +410,9 @@ class InworldHttpTTSService(TTSService):
word_times, chunk_end_time = self._calculate_word_times(timestamp_info)
if word_times:
self._current_run_had_timestamps = True
await self.add_word_timestamps(word_times, context_id)
await self.add_word_timestamps(
word_times, context_id, includes_inter_frame_spaces=True
)
# Track the maximum end time across all chunks
utterance_duration = max(utterance_duration, chunk_end_time)
@@ -447,7 +449,9 @@ class InworldHttpTTSService(TTSService):
word_times, chunk_end_time = self._calculate_word_times(timestamp_info)
if word_times:
self._current_run_had_timestamps = True
await self.add_word_timestamps(word_times, context_id)
await self.add_word_timestamps(
word_times, context_id, includes_inter_frame_spaces=True
)
utterance_duration = chunk_end_time
audio_data = base64.b64decode(response_data["audioContent"])
@@ -1013,7 +1017,9 @@ class InworldTTSService(WebsocketTTSService):
if word_times:
if ctx_id:
self._contexts_with_timestamps.add(ctx_id)
await self.add_word_timestamps(word_times, ctx_id)
await self.add_word_timestamps(
word_times, ctx_id, includes_inter_frame_spaces=True
)
# Handle flush completion, which indicates the end of a generation
if "flushCompleted" in result:

38
src/pipecat/services/tts_service.py
View File

@@ -96,6 +96,7 @@ class _WordTimestampEntry:
word: str
timestamp: float
context_id: str
includes_inter_frame_spaces: bool = False
class TTSService(AIService):
@@ -1049,8 +1050,10 @@ class TTSService(AIService):
if self._initial_word_times:
cached = self._initial_word_times.copy()
self._initial_word_times = []
for word, timestamp_seconds, ctx_id in cached:
await self._add_word_timestamps([(word, timestamp_seconds)], ctx_id)
for word, timestamp_seconds, ctx_id, ifs in cached:
await self._add_word_timestamps(
[(word, timestamp_seconds)], ctx_id, includes_inter_frame_spaces=ifs
)
async def reset_word_timestamps(self):
"""Reset word timestamp tracking."""
@@ -1060,7 +1063,10 @@ class TTSService(AIService):
self._initial_word_times = []
async def add_word_timestamps(
self, word_times: list[tuple[str, float]], context_id: str | None = None
self,
word_times: list[tuple[str, float]],
context_id: str | None = None,
includes_inter_frame_spaces: bool = False,
):
"""Add word timestamps for processing.
@@ -1072,6 +1078,9 @@ class TTSService(AIService):
Args:
word_times: List of (word, timestamp) tuples where timestamp is in seconds.
context_id: Unique identifier for the TTS context.
includes_inter_frame_spaces: When True, the tokens already embed inter-word
spacing (spaces and punctuation are part of the token text). Downstream
consumers must not inject additional spaces between tokens.
"""
if context_id and self.audio_context_available(context_id):
for word, timestamp in word_times:
@@ -1081,13 +1090,21 @@ class TTSService(AIService):
word=word,
timestamp=timestamp,
context_id=context_id,
includes_inter_frame_spaces=includes_inter_frame_spaces,
),
)
else:
await self._add_word_timestamps(word_times=word_times, context_id=context_id)
await self._add_word_timestamps(
word_times=word_times,
context_id=context_id,
includes_inter_frame_spaces=includes_inter_frame_spaces,
)
async def _add_word_timestamps(
self, word_times: list[tuple[str, float]], context_id: str | None = None
self,
word_times: list[tuple[str, float]],
context_id: str | None = None,
includes_inter_frame_spaces: bool = False,
):
"""Process word timestamps directly, building and pushing TTSTextFrames inline.
@@ -1103,11 +1120,12 @@ class TTSService(AIService):
ts_ns = seconds_to_nanoseconds(timestamp)
if self._initial_word_timestamp == -1:
# Cache until we have audio and can compute PTS.
self._initial_word_times.append((word, timestamp, context_id))
self._initial_word_times.append(
(word, timestamp, context_id, includes_inter_frame_spaces)
)
else:
# Assumption: word-by-word text frames don't include spaces, so
# we can rely on the default includes_inter_frame_spaces=False
frame = TTSTextFrame(word, aggregated_by=AggregationType.WORD)
frame.includes_inter_frame_spaces = includes_inter_frame_spaces
frame.pts = self._initial_word_timestamp + ts_ns
frame.context_id = context_id
if context_id in self._tts_contexts:
@@ -1310,7 +1328,9 @@ class TTSService(AIService):
# Route word timestamps through _add_word_timestamps so they are
# processed in playback order alongside audio frames.
await self._add_word_timestamps(
[(frame.word, frame.timestamp)], frame.context_id
[(frame.word, frame.timestamp)],
frame.context_id,
includes_inter_frame_spaces=frame.includes_inter_frame_spaces,
)
continue
elif isinstance(frame, TTSAudioRawFrame):