From 7de8838debbc516cf18cad263375a6eeb339ab41 Mon Sep 17 00:00:00 2001 From: ivaaan Date: Mon, 17 Nov 2025 13:25:12 +0100 Subject: [PATCH 01/14] add word-level timestamp support to Hume service --- src/pipecat/services/hume/tts.py | 45 +++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/hume/tts.py b/src/pipecat/services/hume/tts.py index 6760c8121..b29215727 100644 --- a/src/pipecat/services/hume/tts.py +++ b/src/pipecat/services/hume/tts.py @@ -19,7 +19,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) -from pipecat.services.tts_service import TTSService +from pipecat.services.tts_service import WordTTSService from pipecat.utils.tracing.service_decorators import traced_tts try: @@ -28,6 +28,7 @@ try: FormatPcm, PostedUtterance, PostedUtteranceVoiceWithId, + TimestampType, ) except ModuleNotFoundError as e: # pragma: no cover - import-time guidance logger.error(f"Exception: {e}") @@ -38,7 +39,7 @@ except ModuleNotFoundError as e: # pragma: no cover - import-time guidance HUME_SAMPLE_RATE = 48_000 # Hume TTS streams at 48 kHz -class HumeTTSService(TTSService): +class HumeTTSService(WordTTSService): """Hume Octave Text-to-Speech service. Streams PCM audio via Hume's HTTP output streaming (JSON chunks) endpoint @@ -48,6 +49,7 @@ class HumeTTSService(TTSService): - Generates speech from text using Hume TTS. - Streams PCM audio. + - Supports word timestamps for synchronization with text. - Supports dynamic updates of voice and synthesis parameters at runtime. - Provides metrics for Time To First Byte (TTFB) and TTS usage. """ @@ -85,7 +87,9 @@ class HumeTTSService(TTSService): """ api_key = api_key or os.getenv("HUME_API_KEY") if not api_key: - raise ValueError("HumeTTSService requires an API key (env HUME_API_KEY or api_key=)") + raise ValueError( + "HumeTTSService requires an API key (env HUME_API_KEY or api_key=)" + ) if sample_rate != HUME_SAMPLE_RATE: logger.warning( @@ -101,6 +105,7 @@ class HumeTTSService(TTSService): self.set_voice(voice_id) self._audio_bytes = b"" + self._first_audio_chunk = True def can_generate_metrics(self) -> bool: """Can generate metrics. @@ -193,6 +198,7 @@ class HumeTTSService(TTSService): # Hume emits mono PCM at 48 kHz; downstream can resample if needed. # We buffer audio bytes before sending to prevent glitches. self._audio_bytes = b"" + self._first_audio_chunk = True # Use version "2" by default if no description is provided # Version "1" is needed when description is used @@ -202,11 +208,42 @@ class HumeTTSService(TTSService): format=pcm_fmt, instant_mode=True, version=version, + include_timestamp_types=[TimestampType.WORD], ): + # Check if this is a timestamp chunk + chunk_type = getattr(chunk, "type", None) + if chunk_type == "timestamp": + # Start word timestamps if we haven't received audio yet + if self._first_audio_chunk: + await self.stop_ttfb_metrics() + self.start_word_timestamps() + self._first_audio_chunk = False + # Process word timestamp + timestamp = getattr(chunk, "timestamp", None) + if timestamp: + word_text = getattr(timestamp, "text", None) + time_obj = getattr(timestamp, "time", None) + if word_text and time_obj: + # Convert milliseconds to seconds + begin_ms = getattr(time_obj, "begin", None) + if begin_ms is not None: + begin_seconds = begin_ms / 1000.0 + await self.add_word_timestamps( + [(word_text, begin_seconds)] + ) + continue + + # Process audio chunk audio_b64 = getattr(chunk, "audio", None) if not audio_b64: continue + # Start word timestamps on first audio chunk + if self._first_audio_chunk: + await self.stop_ttfb_metrics() + self.start_word_timestamps() + self._first_audio_chunk = False + pcm_bytes = base64.b64decode(audio_b64) self._audio_bytes += pcm_bytes @@ -230,4 +267,6 @@ class HumeTTSService(TTSService): finally: # Ensure TTFB timer is stopped even on early failures await self.stop_ttfb_metrics() + # Signal end of word timestamps + await self.add_word_timestamps([("TTSStoppedFrame", 0), ("Reset", 0)]) yield TTSStoppedFrame() From 2f2bde9856cc13b060cecd34a60cc75d106906ae Mon Sep 17 00:00:00 2001 From: ivaaan Date: Mon, 17 Nov 2025 13:40:03 +0100 Subject: [PATCH 02/14] add timestamps to example --- .../foundational/07ae-interruptible-hume.py | 49 +++++++++++++++++-- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/examples/foundational/07ae-interruptible-hume.py b/examples/foundational/07ae-interruptible-hume.py index 046f2d4c8..501eafad3 100644 --- a/examples/foundational/07ae-interruptible-hume.py +++ b/examples/foundational/07ae-interruptible-hume.py @@ -13,13 +13,17 @@ from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame +from pipecat.frames.frames import Frame, LLMRunFrame, TTSTextFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, +) +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor +from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -28,9 +32,25 @@ from pipecat.services.openai.llm import OpenAILLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams +from pipecat.frames.frames import format_pts load_dotenv(override=True) + +class TimestampLogger(FrameProcessor): + """Frame processor that logs TTSTextFrame objects with their timestamps. + + This helps verify that word timestamps are working correctly by showing + when each word is spoken with its presentation timestamp (PTS). + """ + + async def process_frame(self, frame: Frame, direction: FrameDirection): + if isinstance(frame, TTSTextFrame): + pts_str = format_pts(frame.pts) if frame.pts else "no PTS" + logger.info(f"🎯 Word timestamp: '{frame.text}' at {pts_str}") + await self.push_frame(frame, direction) + + # We store functions so objects (e.g. SileroVADAnalyzer) don't get # instantiated. The function will be called when the desired transport gets # selected. @@ -81,15 +101,24 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): rtvi = RTVIProcessor(config=RTVIConfig(config=[])) + # Add transcript processor to show timestamps in conversation history + transcript = TranscriptProcessor() + + # Add timestamp logger to verify word timestamps are being generated + timestamp_logger = TimestampLogger() + pipeline = Pipeline( [ transport.input(), # Transport user input rtvi, stt, + transcript.user(), # User transcripts context_aggregator.user(), # User responses llm, # LLM - tts, # TTS + tts, # TTS (HumeTTSService with word timestamps) + timestamp_logger, # Log word timestamps for verification transport.output(), # Transport bot output + transcript.assistant(), # Assistant transcripts with timestamps context_aggregator.assistant(), # Assistant spoken responses ] ) @@ -109,11 +138,23 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): async def on_client_ready(rtvi): await rtvi.set_bot_ready() + @transcript.event_handler("on_transcript_update") + async def on_transcript_update(processor, frame): + """Log transcript updates to show timestamps in conversation.""" + for msg in frame.messages: + timestamp_str = f"[{msg.timestamp}] " if msg.timestamp else "" + logger.info(f"📝 Transcript: {timestamp_str}{msg.role}: {msg.content}") + @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info(f"Client connected") + logger.info( + "💡 Word timestamps are enabled! Watch for '🎯 Word timestamp' logs showing each word with its PTS." + ) # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + messages.append( + {"role": "system", "content": "Please introduce yourself to the user."} + ) await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") From 71869a116d06c2f120de059a242c319cbbbf8a0f Mon Sep 17 00:00:00 2001 From: ivaaan Date: Mon, 17 Nov 2025 13:51:04 +0100 Subject: [PATCH 03/14] fix errors --- examples/foundational/07ae-interruptible-hume.py | 4 ++++ src/pipecat/services/hume/tts.py | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/foundational/07ae-interruptible-hume.py b/examples/foundational/07ae-interruptible-hume.py index 501eafad3..17a29f0db 100644 --- a/examples/foundational/07ae-interruptible-hume.py +++ b/examples/foundational/07ae-interruptible-hume.py @@ -45,9 +45,13 @@ class TimestampLogger(FrameProcessor): """ async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, TTSTextFrame): pts_str = format_pts(frame.pts) if frame.pts else "no PTS" logger.info(f"🎯 Word timestamp: '{frame.text}' at {pts_str}") + + # Always push all frames through await self.push_frame(frame, direction) diff --git a/src/pipecat/services/hume/tts.py b/src/pipecat/services/hume/tts.py index b29215727..0ee6e2adf 100644 --- a/src/pipecat/services/hume/tts.py +++ b/src/pipecat/services/hume/tts.py @@ -28,7 +28,6 @@ try: FormatPcm, PostedUtterance, PostedUtteranceVoiceWithId, - TimestampType, ) except ModuleNotFoundError as e: # pragma: no cover - import-time guidance logger.error(f"Exception: {e}") @@ -208,7 +207,7 @@ class HumeTTSService(WordTTSService): format=pcm_fmt, instant_mode=True, version=version, - include_timestamp_types=[TimestampType.WORD], + include_timestamp_types=["word"], ): # Check if this is a timestamp chunk chunk_type = getattr(chunk, "type", None) From 9156e217275684231a9d29c75750ea616744360e Mon Sep 17 00:00:00 2001 From: ivaaan Date: Mon, 17 Nov 2025 14:00:03 +0100 Subject: [PATCH 04/14] fix formatting --- examples/foundational/07ae-interruptible-hume.py | 7 ++----- src/pipecat/services/hume/tts.py | 8 ++------ 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/examples/foundational/07ae-interruptible-hume.py b/examples/foundational/07ae-interruptible-hume.py index 17a29f0db..e30cfff98 100644 --- a/examples/foundational/07ae-interruptible-hume.py +++ b/examples/foundational/07ae-interruptible-hume.py @@ -13,7 +13,7 @@ from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import Frame, LLMRunFrame, TTSTextFrame +from pipecat.frames.frames import Frame, LLMRunFrame, TTSTextFrame, format_pts from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -32,7 +32,6 @@ from pipecat.services.openai.llm import OpenAILLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams -from pipecat.frames.frames import format_pts load_dotenv(override=True) @@ -156,9 +155,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): "💡 Word timestamps are enabled! Watch for '🎯 Word timestamp' logs showing each word with its PTS." ) # Kick off the conversation. - messages.append( - {"role": "system", "content": "Please introduce yourself to the user."} - ) + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") diff --git a/src/pipecat/services/hume/tts.py b/src/pipecat/services/hume/tts.py index 0ee6e2adf..c3ef18ddf 100644 --- a/src/pipecat/services/hume/tts.py +++ b/src/pipecat/services/hume/tts.py @@ -86,9 +86,7 @@ class HumeTTSService(WordTTSService): """ api_key = api_key or os.getenv("HUME_API_KEY") if not api_key: - raise ValueError( - "HumeTTSService requires an API key (env HUME_API_KEY or api_key=)" - ) + raise ValueError("HumeTTSService requires an API key (env HUME_API_KEY or api_key=)") if sample_rate != HUME_SAMPLE_RATE: logger.warning( @@ -227,9 +225,7 @@ class HumeTTSService(WordTTSService): begin_ms = getattr(time_obj, "begin", None) if begin_ms is not None: begin_seconds = begin_ms / 1000.0 - await self.add_word_timestamps( - [(word_text, begin_seconds)] - ) + await self.add_word_timestamps([(word_text, begin_seconds)]) continue # Process audio chunk From 2a51d0f1e5d4cb1546e214e3f3d75b9cacc8cd50 Mon Sep 17 00:00:00 2001 From: ivaaan Date: Mon, 17 Nov 2025 15:20:06 +0100 Subject: [PATCH 05/14] add changelog --- CHANGELOG.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cda253c7..baed81a18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 services that subclass `TTSService` can indicate whether the text in the `TTSTextFrame`s they push already contain any necessary inter-frame spaces. +- Added word-level timestamps support to Hume TTS service + ### Changed - Updated all STT and TTS services to use consistent error handling pattern with @@ -56,8 +58,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and - example wiring; leverages the enhancement model for robust detection with no +- Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and + example wiring; leverages the enhancement model for robust detection with no ONNX dependency or added processing complexity. ## [0.0.94] - 2025-11-10 From 9944e6faf0447f3034e5a3c7e95db0ef1ec387e5 Mon Sep 17 00:00:00 2001 From: ivaaan Date: Tue, 18 Nov 2025 18:25:53 +0100 Subject: [PATCH 06/14] upd service based on Mark's suggestions --- src/pipecat/services/hume/tts.py | 126 ++++++++++++++++++++----------- 1 file changed, 81 insertions(+), 45 deletions(-) diff --git a/src/pipecat/services/hume/tts.py b/src/pipecat/services/hume/tts.py index c3ef18ddf..027a5851b 100644 --- a/src/pipecat/services/hume/tts.py +++ b/src/pipecat/services/hume/tts.py @@ -14,11 +14,13 @@ from pydantic import BaseModel from pipecat.frames.frames import ( ErrorFrame, Frame, + InterruptionFrame, StartFrame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.processors.frame_processor import FrameDirection from pipecat.services.tts_service import WordTTSService from pipecat.utils.tracing.service_decorators import traced_tts @@ -29,6 +31,7 @@ try: PostedUtterance, PostedUtteranceVoiceWithId, ) + from hume.tts.types import TimestampMessage except ModuleNotFoundError as e: # pragma: no cover - import-time guidance logger.error(f"Exception: {e}") logger.error("In order to use Hume, you need to `pip install pipecat-ai[hume]`.") @@ -48,7 +51,7 @@ class HumeTTSService(WordTTSService): - Generates speech from text using Hume TTS. - Streams PCM audio. - - Supports word timestamps for synchronization with text. + - Supports word-level timestamps for precise audio-text synchronization. - Supports dynamic updates of voice and synthesis parameters at runtime. - Provides metrics for Time To First Byte (TTFB) and TTS usage. """ @@ -93,7 +96,12 @@ class HumeTTSService(WordTTSService): f"Hume TTS streams at {HUME_SAMPLE_RATE} Hz; configured sample_rate={sample_rate}" ) - super().__init__(sample_rate=sample_rate, **kwargs) + # WordTTSService sets push_text_frames=False by default, which we want + super().__init__( + sample_rate=sample_rate, + push_text_frames=False, + **kwargs, + ) self._client = AsyncHumeClient(api_key=api_key) self._params = params or HumeTTSService.InputParams() @@ -102,7 +110,10 @@ class HumeTTSService(WordTTSService): self.set_voice(voice_id) self._audio_bytes = b"" - self._first_audio_chunk = True + + # Track cumulative time for word timestamps across utterances + self._cumulative_time = 0.0 + self._started = False def can_generate_metrics(self) -> bool: """Can generate metrics. @@ -128,6 +139,27 @@ class HumeTTSService(WordTTSService): frame: The start frame. """ await super().start(frame) + self._reset_state() + + def _reset_state(self): + """Reset internal state variables.""" + self._cumulative_time = 0.0 + self._started = False + + async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): + """Push a frame and handle state changes. + + Args: + frame: The frame to push. + direction: The direction to push the frame. + """ + await super().push_frame(frame, direction) + if isinstance(frame, (InterruptionFrame, TTSStoppedFrame)): + # Reset timing on interruption or stop + self._reset_state() + + if isinstance(frame, TTSStoppedFrame): + await self.add_word_timestamps([("Reset", 0)]) async def update_setting(self, key: str, value: Any) -> None: """Runtime updates via `TTSUpdateSettingsFrame`. @@ -144,7 +176,7 @@ class HumeTTSService(WordTTSService): if key_l == "voice_id": self.set_voice(str(value)) - logger.info(f"HumeTTSService voice_id set to: {self.voice}") + logger.debug(f"HumeTTSService voice_id set to: {self.voice}") elif key_l == "description": self._params.description = None if value is None else str(value) elif key_l == "speed": @@ -157,7 +189,7 @@ class HumeTTSService(WordTTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - """Generate speech from text using Hume TTS. + """Generate speech from text using Hume TTS with word timestamps. Args: text: The text to be synthesized. @@ -188,64 +220,66 @@ class HumeTTSService(WordTTSService): await self.start_ttfb_metrics() await self.start_tts_usage_metrics(text) - yield TTSStartedFrame() + + # Start TTS sequence if not already started + if not self._started: + self.start_word_timestamps() + yield TTSStartedFrame() + self._started = True try: # Instant mode is always enabled here (not user-configurable) # Hume emits mono PCM at 48 kHz; downstream can resample if needed. # We buffer audio bytes before sending to prevent glitches. self._audio_bytes = b"" - self._first_audio_chunk = True # Use version "2" by default if no description is provided # Version "1" is needed when description is used version = "1" if self._params.description is not None else "2" + + # Track the duration of this utterance based on the last timestamp + utterance_duration = 0.0 + async for chunk in self._client.tts.synthesize_json_streaming( utterances=[utterance], format=pcm_fmt, instant_mode=True, version=version, - include_timestamp_types=["word"], + include_timestamp_types=["word"], # Request word-level timestamps ): - # Check if this is a timestamp chunk - chunk_type = getattr(chunk, "type", None) - if chunk_type == "timestamp": - # Start word timestamps if we haven't received audio yet - if self._first_audio_chunk: - await self.stop_ttfb_metrics() - self.start_word_timestamps() - self._first_audio_chunk = False - # Process word timestamp - timestamp = getattr(chunk, "timestamp", None) - if timestamp: - word_text = getattr(timestamp, "text", None) - time_obj = getattr(timestamp, "time", None) - if word_text and time_obj: - # Convert milliseconds to seconds - begin_ms = getattr(time_obj, "begin", None) - if begin_ms is not None: - begin_seconds = begin_ms / 1000.0 - await self.add_word_timestamps([(word_text, begin_seconds)]) - continue - - # Process audio chunk + # Process audio chunks audio_b64 = getattr(chunk, "audio", None) - if not audio_b64: - continue - - # Start word timestamps on first audio chunk - if self._first_audio_chunk: + if audio_b64: await self.stop_ttfb_metrics() - self.start_word_timestamps() - self._first_audio_chunk = False + pcm_bytes = base64.b64decode(audio_b64) + self._audio_bytes += pcm_bytes - pcm_bytes = base64.b64decode(audio_b64) - self._audio_bytes += pcm_bytes + # Buffer audio until we have enough to avoid glitches + if len(self._audio_bytes) >= self.chunk_size: + frame = TTSAudioRawFrame( + audio=self._audio_bytes, + sample_rate=self.sample_rate, + num_channels=1, + ) + yield frame + self._audio_bytes = b"" - # Buffer audio until we have enough to avoid glitches - if len(self._audio_bytes) < self.chunk_size: - continue + # Process timestamp messages + if isinstance(chunk, TimestampMessage): + timestamp = chunk.timestamp + if timestamp.type == "word": + # Convert milliseconds to seconds and add cumulative offset + word_start_time = self._cumulative_time + (timestamp.time.begin / 1000.0) + word_end_time = self._cumulative_time + (timestamp.time.end / 1000.0) + # Track the maximum end time for this utterance + utterance_duration = max(utterance_duration, word_end_time) + + # Add word timestamp + await self.add_word_timestamps([(timestamp.text, word_start_time)]) + + # Flush any remaining audio bytes + if self._audio_bytes: frame = TTSAudioRawFrame( audio=self._audio_bytes, sample_rate=self.sample_rate, @@ -256,12 +290,14 @@ class HumeTTSService(WordTTSService): self._audio_bytes = b"" + # Update cumulative time for next utterance + if utterance_duration > 0: + self._cumulative_time = utterance_duration + except Exception as e: logger.error(f"{self} exception: {e}") await self.push_error(ErrorFrame(error=f"{self} error: {e}")) finally: # Ensure TTFB timer is stopped even on early failures await self.stop_ttfb_metrics() - # Signal end of word timestamps - await self.add_word_timestamps([("TTSStoppedFrame", 0), ("Reset", 0)]) - yield TTSStoppedFrame() + # Let the parent class handle TTSStoppedFrame via push_stop_frames From 26f96d0be8e8d1813157bb92b0b99c1b04535a28 Mon Sep 17 00:00:00 2001 From: ivaaan Date: Tue, 18 Nov 2025 18:31:38 +0100 Subject: [PATCH 07/14] upd example --- .../foundational/07ae-interruptible-hume.py | 38 ++++++------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/examples/foundational/07ae-interruptible-hume.py b/examples/foundational/07ae-interruptible-hume.py index e30cfff98..eae9edaa9 100644 --- a/examples/foundational/07ae-interruptible-hume.py +++ b/examples/foundational/07ae-interruptible-hume.py @@ -13,7 +13,8 @@ from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import Frame, LLMRunFrame, TTSTextFrame, format_pts +from pipecat.frames.frames import LLMRunFrame, TTSTextFrame +from pipecat.observers.loggers.debug_log_observer import DebugLogObserver, FrameEndpoint from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -21,7 +22,6 @@ from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import ( LLMContextAggregatorPair, ) -from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments @@ -29,6 +29,7 @@ from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.hume.tts import HUME_SAMPLE_RATE, HumeTTSService from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams @@ -36,24 +37,6 @@ from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams load_dotenv(override=True) -class TimestampLogger(FrameProcessor): - """Frame processor that logs TTSTextFrame objects with their timestamps. - - This helps verify that word timestamps are working correctly by showing - when each word is spoken with its presentation timestamp (PTS). - """ - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - if isinstance(frame, TTSTextFrame): - pts_str = format_pts(frame.pts) if frame.pts else "no PTS" - logger.info(f"🎯 Word timestamp: '{frame.text}' at {pts_str}") - - # Always push all frames through - await self.push_frame(frame, direction) - - # We store functions so objects (e.g. SileroVADAnalyzer) don't get # instantiated. The function will be called when the desired transport gets # selected. @@ -107,9 +90,6 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Add transcript processor to show timestamps in conversation history transcript = TranscriptProcessor() - # Add timestamp logger to verify word timestamps are being generated - timestamp_logger = TimestampLogger() - pipeline = Pipeline( [ transport.input(), # Transport user input @@ -119,7 +99,6 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): context_aggregator.user(), # User responses llm, # LLM tts, # TTS (HumeTTSService with word timestamps) - timestamp_logger, # Log word timestamps for verification transport.output(), # Transport bot output transcript.assistant(), # Assistant transcripts with timestamps context_aggregator.assistant(), # Assistant spoken responses @@ -134,7 +113,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): audio_out_sample_rate=HUME_SAMPLE_RATE, ), idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - observers=[RTVIObserver(rtvi)], + observers=[ + RTVIObserver(rtvi), + DebugLogObserver( + frame_types={ + TTSTextFrame: (BaseOutputTransport, FrameEndpoint.DESTINATION), + } + ), + ], ) @rtvi.event_handler("on_client_ready") @@ -152,7 +138,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): async def on_client_connected(transport, client): logger.info(f"Client connected") logger.info( - "💡 Word timestamps are enabled! Watch for '🎯 Word timestamp' logs showing each word with its PTS." + "💡 Word timestamps are enabled! Watch the console for TTSTextFrame logs showing each word with its PTS." ) # Kick off the conversation. messages.append({"role": "system", "content": "Please introduce yourself to the user."}) From a38f20813505665c03f07b15d8f7ad07d4e36b8b Mon Sep 17 00:00:00 2001 From: Ivan A Date: Tue, 18 Nov 2025 20:30:28 +0100 Subject: [PATCH 08/14] Update examples/foundational/07ae-interruptible-hume.py Co-authored-by: Mark Backman --- examples/foundational/07ae-interruptible-hume.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/foundational/07ae-interruptible-hume.py b/examples/foundational/07ae-interruptible-hume.py index eae9edaa9..9233e58dd 100644 --- a/examples/foundational/07ae-interruptible-hume.py +++ b/examples/foundational/07ae-interruptible-hume.py @@ -117,7 +117,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): RTVIObserver(rtvi), DebugLogObserver( frame_types={ - TTSTextFrame: (BaseOutputTransport, FrameEndpoint.DESTINATION), + TTSTextFrame: (BaseOutputTransport, FrameEndpoint.SOURCE), } ), ], From 4ae1819645f2b6ef4b9f925ded629946b962d90e Mon Sep 17 00:00:00 2001 From: Ivan A Date: Tue, 18 Nov 2025 20:30:44 +0100 Subject: [PATCH 09/14] Update src/pipecat/services/hume/tts.py Co-authored-by: Mark Backman --- src/pipecat/services/hume/tts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pipecat/services/hume/tts.py b/src/pipecat/services/hume/tts.py index bba70a759..278626748 100644 --- a/src/pipecat/services/hume/tts.py +++ b/src/pipecat/services/hume/tts.py @@ -100,6 +100,7 @@ class HumeTTSService(WordTTSService): super().__init__( sample_rate=sample_rate, push_text_frames=False, + push_stop_frames=True, **kwargs, ) From c2309efd7eea91779ad4e8c829e8237ca88eb17e Mon Sep 17 00:00:00 2001 From: ivaaan Date: Tue, 18 Nov 2025 20:32:27 +0100 Subject: [PATCH 10/14] rm TranscriptProcessor --- examples/foundational/07ae-interruptible-hume.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/examples/foundational/07ae-interruptible-hume.py b/examples/foundational/07ae-interruptible-hume.py index 9233e58dd..a7bea804c 100644 --- a/examples/foundational/07ae-interruptible-hume.py +++ b/examples/foundational/07ae-interruptible-hume.py @@ -23,7 +23,6 @@ from pipecat.processors.aggregators.llm_response_universal import ( LLMContextAggregatorPair, ) from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor -from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -87,9 +86,6 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): rtvi = RTVIProcessor(config=RTVIConfig(config=[])) - # Add transcript processor to show timestamps in conversation history - transcript = TranscriptProcessor() - pipeline = Pipeline( [ transport.input(), # Transport user input From 4c3fd42b1ccea1be1ff6153398bb5b2e199dd953 Mon Sep 17 00:00:00 2001 From: ivaaan Date: Tue, 18 Nov 2025 20:36:45 +0100 Subject: [PATCH 11/14] fix changelog --- CHANGELOG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83796864b..b346e002d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -90,8 +90,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Prevented `HeyGenVideoService` from automatically disconnecting after 5 minutes. -### Added - - Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and example wiring; leverages the enhancement model for robust detection with no ONNX dependency or added processing complexity. From f325eeb95baf01b8cf54421c0e21dde932fb85ad Mon Sep 17 00:00:00 2001 From: ivaaan Date: Tue, 18 Nov 2025 20:41:10 +0100 Subject: [PATCH 12/14] rm TranscriptProcessor 2 --- examples/foundational/07ae-interruptible-hume.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/examples/foundational/07ae-interruptible-hume.py b/examples/foundational/07ae-interruptible-hume.py index a7bea804c..c5de34c85 100644 --- a/examples/foundational/07ae-interruptible-hume.py +++ b/examples/foundational/07ae-interruptible-hume.py @@ -91,12 +91,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): transport.input(), # Transport user input rtvi, stt, - transcript.user(), # User transcripts context_aggregator.user(), # User responses llm, # LLM tts, # TTS (HumeTTSService with word timestamps) transport.output(), # Transport bot output - transcript.assistant(), # Assistant transcripts with timestamps context_aggregator.assistant(), # Assistant spoken responses ] ) @@ -123,13 +121,6 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): async def on_client_ready(rtvi): await rtvi.set_bot_ready() - @transcript.event_handler("on_transcript_update") - async def on_transcript_update(processor, frame): - """Log transcript updates to show timestamps in conversation.""" - for msg in frame.messages: - timestamp_str = f"[{msg.timestamp}] " if msg.timestamp else "" - logger.info(f"📝 Transcript: {timestamp_str}{msg.role}: {msg.content}") - @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info(f"Client connected") From 771469b834b25e1d679d5de80cceee2e77a2d23b Mon Sep 17 00:00:00 2001 From: ivaaan Date: Tue, 18 Nov 2025 21:39:29 +0100 Subject: [PATCH 13/14] fix changelog --- CHANGELOG.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b346e002d..3806c2062 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,10 +22,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `ElevenLabsRealtimeSTTService` which implements the Realtime STT service from ElevenLabs. -- Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and - example wiring; leverages the enhancement model for robust detection with no - ONNX dependency or added processing complexity. - - Added word-level timestamps support to Hume TTS service ### Changed From 6484855139dc0d5265bd3210aff9dcc146717645 Mon Sep 17 00:00:00 2001 From: ivaaan Date: Tue, 18 Nov 2025 21:47:46 +0100 Subject: [PATCH 14/14] fix changelog --- CHANGELOG.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3806c2062..47be7d2bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -86,10 +86,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Prevented `HeyGenVideoService` from automatically disconnecting after 5 minutes. -- Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and - example wiring; leverages the enhancement model for robust detection with no - ONNX dependency or added processing complexity. - ## [0.0.94] - 2025-11-10 ### Changed