diff --git a/CHANGELOG.md b/CHANGELOG.md index 10aa1c662..d69eb897a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.0.95] - 2025-11-18 ### Added @@ -22,15 +22,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `ElevenLabsRealtimeSTTService` which implements the Realtime STT service from ElevenLabs. -- Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and - example wiring; leverages the enhancement model for robust detection with no - ONNX dependency or added processing complexity. +- Added word-level timestamps support to Hume TTS service ### Changed -- ⚠️ Breaking change: `LLMContext.create_image_message()` and - `LLMContext.create_audio_message()` are now async methods. This fixes and - issue where the asyncio event loop would be blocked while encoding audio or +- ⚠️ Breaking change: `LLMContext.create_image_message()`, + `LLMContext.create_audio_message()`, `LLMContext.add_image_frame_message()` + and `LLMContext.add_audio_frames_message()` are now async methods. This fixes + an issue where the asyncio event loop would be blocked while encoding audio or images. - `ConsumerProcessor` now queues frames from the producer internally instead of @@ -71,10 +70,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- Fixed a race condition where, if the LLM received instructions to both produce - text and invoke a function call at the same time, the context would not be - updated before the function call result arrived, causing the bot to repeat - itself. +- Fixed a `SimliVideoService` connection issue. - Fixed an issue in the `Runner` where, when using `SmallWebRTCTransport`, the `request_data` was not being passed to the `SmallWebRTCRunnerArguments` body. diff --git a/examples/foundational/07ae-interruptible-hume.py b/examples/foundational/07ae-interruptible-hume.py index 046f2d4c8..c5de34c85 100644 --- a/examples/foundational/07ae-interruptible-hume.py +++ b/examples/foundational/07ae-interruptible-hume.py @@ -13,24 +13,29 @@ from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame +from pipecat.frames.frames import LLMRunFrame, TTSTextFrame +from pipecat.observers.loggers.debug_log_observer import DebugLogObserver, FrameEndpoint from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, +) from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.hume.tts import HUME_SAMPLE_RATE, HumeTTSService from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams load_dotenv(override=True) + # We store functions so objects (e.g. SileroVADAnalyzer) don't get # instantiated. The function will be called when the desired transport gets # selected. @@ -88,7 +93,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): stt, context_aggregator.user(), # User responses llm, # LLM - tts, # TTS + tts, # TTS (HumeTTSService with word timestamps) transport.output(), # Transport bot output context_aggregator.assistant(), # Assistant spoken responses ] @@ -102,7 +107,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): audio_out_sample_rate=HUME_SAMPLE_RATE, ), idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - observers=[RTVIObserver(rtvi)], + observers=[ + RTVIObserver(rtvi), + DebugLogObserver( + frame_types={ + TTSTextFrame: (BaseOutputTransport, FrameEndpoint.SOURCE), + } + ), + ], ) @rtvi.event_handler("on_client_ready") @@ -112,6 +124,9 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info(f"Client connected") + logger.info( + "💡 Word timestamps are enabled! Watch the console for TTSTextFrame logs showing each word with its PTS." + ) # Kick off the conversation. messages.append({"role": "system", "content": "Please introduce yourself to the user."}) await task.queue_frames([LLMRunFrame()]) diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 7a7155297..a7837ce60 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -391,7 +391,7 @@ class AudioAccumulator(FrameProcessor): ) self._user_speaking = False context = LLMContext() - context.add_audio_frames_message(audio_frames=self._audio_frames) + await context.add_audio_frames_message(audio_frames=self._audio_frames) await self.push_frame(LLMContextFrame(context=context)) elif isinstance(frame, InputAudioRawFrame): # Append the audio frame to our buffer. Treat the buffer as a ring buffer, dropping the oldest diff --git a/examples/foundational/39-mcp-stdio.py b/examples/foundational/39-mcp-stdio.py index b88ee30b1..7127d831f 100644 --- a/examples/foundational/39-mcp-stdio.py +++ b/examples/foundational/39-mcp-stdio.py @@ -155,7 +155,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. You have access to tools to search the Rijksmuseum collection. - Offer, for example, to show the earliest Rembrandt work from the museum. Use the `search_artwork` tool. + Offer, for example, to show a floral still life, use the `search_artwork` tool. The tool may respond with a JSON object with an `artworks` array. Choose the art from that array. Once the tool has responded, tell the user the title and use the `open_image_in_browser` tool. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. diff --git a/examples/foundational/39a-mcp-run-sse.py b/examples/foundational/39a-mcp-run-sse.py deleted file mode 100644 index 328af6ce4..000000000 --- a/examples/foundational/39a-mcp-run-sse.py +++ /dev/null @@ -1,159 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - - -import os - -from dotenv import load_dotenv -from loguru import logger -from mcp.client.session_group import SseServerParameters - -from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams -from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair -from pipecat.runner.types import RunnerArguments -from pipecat.runner.utils import create_transport -from pipecat.services.anthropic.llm import AnthropicLLMService -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.services.mcp_service import MCPClient -from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.transports.daily.transport import DailyParams -from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams - -load_dotenv(override=True) - -# We store functions so objects (e.g. SileroVADAnalyzer) don't get -# instantiated. The function will be called when the desired transport gets -# selected. -transport_params = { - "daily": lambda: DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), - "twilio": lambda: FastAPIWebsocketParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), - "webrtc": lambda: TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), -} - - -async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - logger.info(f"Starting bot") - - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) - - llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-7-sonnet-latest" - ) - - try: - # https://docs.mcp.run/integrating/tutorials/mcp-run-sse-openai-agents/ - mcp = MCPClient(server_params=SseServerParameters(url=os.getenv("MCP_RUN_SSE_URL"))) - except Exception as e: - logger.error(f"error setting up mcp") - logger.exception("error trace:") - - tools = {} - try: - tools = await mcp.register_tools(llm) - except Exception as e: - logger.error(f"error registering tools") - logger.exception("error trace:") - - system = f""" - You are a helpful LLM in a WebRTC call. - Your goal is to demonstrate your capabilities in a succinct way. - You have access to a number of tools provided by mcp.run. Use any and all tools to help users. - Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. - Respond to what the user said in a creative and helpful way. - When asked for today's date, use 'https://www.datetoday.net/'. - Don't overexplain what you are doing. - Just respond with short sentences when you are carrying out tool calls. - """ - - messages = [{"role": "system", "content": system}] - - context = LLMContext(messages, tools) - context_aggregator = LLMContextAggregatorPair(context) - - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, - context_aggregator.user(), # User spoken responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses and tool context - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - enable_usage_metrics=True, - ), - idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - ) - - @transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - logger.info(f"Client connected: {client}") - # Kick off the conversation. - await task.queue_frames([LLMRunFrame()]) - - @transport.event_handler("on_client_disconnected") - async def on_client_disconnected(transport, client): - logger.info(f"Client disconnected") - await task.cancel() - - runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) - - await runner.run(task) - - -async def bot(runner_args: RunnerArguments): - """Main bot entry point compatible with Pipecat Cloud.""" - transport = await create_transport(runner_args, transport_params) - await run_bot(transport, runner_args) - - -if __name__ == "__main__": - if not os.getenv("MCP_RUN_SSE_URL"): - logger.error( - f"Please set MCP_RUN_SSE_URL environment variable for this example. See https://mcp.run" - ) - import sys - - sys.exit(1) - - from pipecat.runner.run import main - - main() diff --git a/examples/foundational/39c-mcp-run-http.py b/examples/foundational/39a-mcp-streamable-http.py similarity index 100% rename from examples/foundational/39c-mcp-run-http.py rename to examples/foundational/39a-mcp-streamable-http.py diff --git a/examples/foundational/39d-mcp-run-http-gemini-live.py b/examples/foundational/39b-mcp-streamable-http-gemini-live.py similarity index 100% rename from examples/foundational/39d-mcp-run-http-gemini-live.py rename to examples/foundational/39b-mcp-streamable-http-gemini-live.py diff --git a/examples/foundational/39b-multiple-mcp.py b/examples/foundational/39c-multiple-mcp.py similarity index 80% rename from examples/foundational/39b-multiple-mcp.py rename to examples/foundational/39c-multiple-mcp.py index dad059208..88fdc3610 100644 --- a/examples/foundational/39b-multiple-mcp.py +++ b/examples/foundational/39c-multiple-mcp.py @@ -7,6 +7,7 @@ import asyncio import io +import json import os import re import shutil @@ -15,7 +16,7 @@ import aiohttp from dotenv import load_dotenv from loguru import logger from mcp import StdioServerParameters -from mcp.client.session_group import SseServerParameters +from mcp.client.session_group import StreamableHttpParameters from PIL import Image from pipecat.adapters.schemas.tools_schema import ToolsSchema @@ -66,10 +67,12 @@ class UrlToImageProcessor(FrameProcessor): await self.push_frame(frame, direction) def extract_url(self, text: str): - pattern = r"!\[[^\]]*\]\((https?://[^)]+\.(png|jpg|jpeg|PNG|JPG|JPEG|gif))\)" - match = re.search(pattern, text) - if match: - return match.group(1) + data = json.loads(text) + if "artObject" in data: + return data["artObject"]["webImage"]["url"] + if "artworks" in data and len(data["artworks"]): + return data["artworks"][0]["webImage"]["url"] + return None async def run_image_process(self, image_url: str): @@ -132,10 +135,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): system = f""" You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. - You have access to tools to search the Rijksmuseum collection. - Offer, for example, to show the earliest Rembrandt work from the museum. Use the `search_artwork` tool. + You have access to tools to search the Rijksmuseum collection and the user's GitHub repositories and account. + Offer, for example, to show a floral still life, use the `search_artwork` tool. The tool may respond with a JSON object with an `artworks` array. Choose the art from that array. Once the tool has responded, tell the user the title and use the `open_image_in_browser` tool. + You can also offer to answer users questions about their GitHub repositories and account. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way. Don't overexplain what you are doing. @@ -145,11 +149,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [{"role": "system", "content": system}] try: - mcp = MCPClient( + rijksmuseum_mcp = MCPClient( server_params=StdioServerParameters( command=shutil.which("npx"), # https://github.com/r-huijts/rijksmuseum-mcp - args=["-y", "mcp-server-error setting up mcp"], + args=["-y", "mcp-server-rijksmuseum"], env={"RIJKSMUSEUM_API_KEY": os.getenv("RIJKSMUSEUM_API_KEY")}, ) ) @@ -157,24 +161,32 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.error(f"error setting up rijksmuseum mcp") logger.exception("error trace:") try: - # https://docs.mcp.run/integrating/tutorials/mcp-run-sse-openai-agents/ - # ie. "https://www.mcp.run/api/mcp/sse?..." - # ensure the profile has a tool or few installed - mcp_run = MCPClient(server_params=SseServerParameters(url=os.getenv("MCP_RUN_SSE_URL"))) + # Github MCP docs: https://github.com/github/github-mcp-server + # Enable Github Copilot on your GitHub account. Free tier is ok. (https://github.com/settings/copilot) + # Generate a personal access token. It must be a Fine-grained token, classic tokens are not supported. (https://github.com/settings/personal-access-tokens) + # Set permissions you want to use (eg. "all repositories", "profile: read/write", etc) + github_mcp = MCPClient( + server_params=StreamableHttpParameters( + url="https://api.githubcopilot.com/mcp/", + headers={ + "Authorization": f"Bearer {os.getenv('GITHUB_PERSONAL_ACCESS_TOKEN')}" + }, + ) + ) except Exception as e: logger.error(f"error setting up mcp.run") logger.exception("error trace:") - tools = {} - run_tools = {} + rijksmuseum_tools = {} + github_tools = {} try: - tools = await mcp.register_tools(llm) - run_tools = await mcp_run.register_tools(llm) + rijksmuseum_tools = await rijksmuseum_mcp.register_tools(llm) + github_tools = await github_mcp.register_tools(llm) except Exception as e: logger.error(f"error registering tools") logger.exception("error trace:") - all_standard_tools = run_tools.standard_tools + tools.standard_tools + all_standard_tools = rijksmuseum_tools.standard_tools + github_tools.standard_tools all_tools = ToolsSchema(standard_tools=all_standard_tools) context = LLMContext(messages, all_tools) @@ -226,9 +238,9 @@ async def bot(runner_args: RunnerArguments): if __name__ == "__main__": - if not os.getenv("RIJKSMUSEUM_API_KEY") or not os.getenv("MCP_RUN_SSE_URL"): + if not os.getenv("RIJKSMUSEUM_API_KEY") or not os.getenv("GITHUB_PERSONAL_ACCESS_TOKEN"): logger.error( - f"Please set RIJKSMUSEUM_API_KEY and MCP_RUN_SSE_URL environment variables. See https://github.com/r-huijts/rijksmuseum-mcp and https://mcp.run" + f"Please set `RIJKSMUSEUM_API_KEY` and `GITHUB_PERSONAL_ACCESS_TOKEN` environment variables. See https://github.com/r-huijts/rijksmuseum-mcp." ) import sys diff --git a/pyproject.toml b/pyproject.toml index e313c789d..9354c2066 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,7 +99,7 @@ local-smart-turn = [ "coremltools>=8.0", "transformers", "torch>=2.5.0,<3", "tor local-smart-turn-v3 = [ "transformers", "onnxruntime>=1.20.1,<2" ] remote-smart-turn = [] silero = [ "onnxruntime>=1.20.1,<2" ] -simli = [ "simli-ai~=0.1.25"] +simli = [ "simli-ai~=1.0.3"] soniox = [ "pipecat-ai[websockets-base]" ] soundfile = [ "soundfile~=0.13.1" ] speechmatics = [ "speechmatics-rt>=0.5.0" ] diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index da6df053d..4f8268d78 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -30,8 +30,8 @@ EVAL_SIMPLE_MATH = EvalConfig( ) EVAL_WEATHER = EvalConfig( - prompt="What's the weather in San Francisco?", - eval="The user says something specific about the current weather in San Francisco, including the degrees.", + prompt="What's the weather in San Francisco (in farhenheit or celsius)?", + eval="The user says something specific about the current weather in San Francisco, including the degrees (in farhenheit or celsius).", ) EVAL_ONLINE_SEARCH = EvalConfig( @@ -70,7 +70,7 @@ EVAL_VOICEMAIL = EvalConfig( EVAL_CONVERSATION = EvalConfig( prompt="Hello, this is Mark.", - eval="The user replies with a greeting.", + eval="The user acknowledges the greeting.", eval_speaks_first=True, ) diff --git a/src/pipecat/processors/aggregators/llm_context.py b/src/pipecat/processors/aggregators/llm_context.py index b9216103a..99b9aeaa9 100644 --- a/src/pipecat/processors/aggregators/llm_context.py +++ b/src/pipecat/processors/aggregators/llm_context.py @@ -180,7 +180,7 @@ class LLMContext: text: Optional text to include with the audio. """ - def encode_audio(): + async def encode_audio(): sample_rate = audio_frames[0].sample_rate num_channels = audio_frames[0].num_channels @@ -198,7 +198,7 @@ class LLMContext: encoded_audio = base64.b64encode(buffer.getvalue()).decode("utf-8") return encoded_audio - encoded_audio = asyncio.to_thread(encode_audio) + encoded_audio = await asyncio.to_thread(encode_audio) content.append( { @@ -333,7 +333,7 @@ class LLMContext: """ self._tool_choice = tool_choice - def add_image_frame_message( + async def add_image_frame_message( self, *, format: str, size: tuple[int, int], image: bytes, text: Optional[str] = None ): """Add a message containing an image frame. @@ -344,10 +344,12 @@ class LLMContext: image: Raw image bytes. text: Optional text to include with the image. """ - message = LLMContext.create_image_message(format=format, size=size, image=image, text=text) + message = await LLMContext.create_image_message( + format=format, size=size, image=image, text=text + ) self.add_message(message) - def add_audio_frames_message( + async def add_audio_frames_message( self, *, audio_frames: list[AudioRawFrame], text: str = "Audio follows" ): """Add a message containing audio frames. @@ -356,7 +358,7 @@ class LLMContext: audio_frames: List of audio frame objects to include. text: Optional text to include with the audio. """ - message = LLMContext.create_audio_message(audio_frames=audio_frames, text=text) + message = await LLMContext.create_audio_message(audio_frames=audio_frames, text=text) self.add_message(message) @staticmethod diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index 8c084f2dc..87974bed2 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -591,8 +591,6 @@ class LLMAssistantAggregator(LLMContextAggregator): self._started = 0 self._function_calls_in_progress: Dict[str, Optional[FunctionCallInProgressFrame]] = {} self._context_updated_tasks: Set[asyncio.Task] = set() - self._function_calls_context_messages = [] - self._function_calls_pending_context_updates_callbacks = [] @property def has_function_calls_in_progress(self) -> bool: @@ -649,23 +647,21 @@ class LLMAssistantAggregator(LLMContextAggregator): async def push_aggregation(self): """Push the current assistant aggregation with timestamp.""" - if self._aggregation: - aggregation = self.aggregation_string() - await self.reset() + if not self._aggregation: + return - if aggregation: - self._context.add_message({"role": "assistant", "content": aggregation}) + aggregation = self.aggregation_string() + await self.reset() - # Push context frame - await self.push_context_frame() + if aggregation: + self._context.add_message({"role": "assistant", "content": aggregation}) - # Push timestamp frame with current time - timestamp_frame = LLMContextAssistantTimestampFrame(timestamp=time_now_iso8601()) - await self.push_frame(timestamp_frame) + # Push context frame + await self.push_context_frame() - if self._function_calls_context_messages: - self._flush_function_call_messages_to_context() - await self.push_context_frame(FrameDirection.UPSTREAM) + # Push timestamp frame with current time + timestamp_frame = LLMContextAssistantTimestampFrame(timestamp=time_now_iso8601()) + await self.push_frame(timestamp_frame) async def _handle_llm_run(self, frame: LLMRunFrame): await self.push_context_frame(FrameDirection.UPSTREAM) @@ -685,23 +681,6 @@ class LLMAssistantAggregator(LLMContextAggregator): self._started = 0 await self.reset() - def _flush_function_call_messages_to_context(self): - """Move all function calls messages into context, then clear the list.""" - if self._function_calls_context_messages: - self._context.add_messages(self._function_calls_context_messages) - self._function_calls_context_messages.clear() - - # Call the `on_context_updated` callbacks once the function call results - # are added to the context. Run them in separate tasks to make - # sure we don't block the pipeline. - for callback, task_name in self._function_calls_pending_context_updates_callbacks: - task = self.create_task(callback(), task_name) - self._context_updated_tasks.add(task) - task.add_done_callback(self._context_updated_task_finished) - - # Clear the pending callbacks list - self._function_calls_pending_context_updates_callbacks.clear() - async def _handle_function_calls_started(self, frame: FunctionCallsStartedFrame): function_names = [f"{f.function_name}:{f.tool_call_id}" for f in frame.function_calls] logger.debug(f"{self} FunctionCallsStartedFrame: {function_names}") @@ -714,7 +693,7 @@ class LLMAssistantAggregator(LLMContextAggregator): ) # Update context with the in-progress function call - self._function_calls_context_messages.append( + self._context.add_message( { "role": "assistant", "tool_calls": [ @@ -729,7 +708,7 @@ class LLMAssistantAggregator(LLMContextAggregator): ], } ) - self._function_calls_context_messages.append( + self._context.add_message( { "role": "tool", "content": "IN_PROGRESS", @@ -760,13 +739,6 @@ class LLMAssistantAggregator(LLMContextAggregator): else: self._update_function_call_result(frame.function_name, frame.tool_call_id, "COMPLETED") - # Store the on_context_updated callback along with task name info to be invoked later - if properties and properties.on_context_updated: - task_name = f"{frame.function_name}:{frame.tool_call_id}:on_context_updated" - self._function_calls_pending_context_updates_callbacks.append( - (properties.on_context_updated, task_name) - ) - run_llm = False # Run inference if the function call result requires it. @@ -781,13 +753,17 @@ class LLMAssistantAggregator(LLMContextAggregator): # If this is the last function call in progress, run the LLM. run_llm = not bool(self._function_calls_in_progress) - # Only run if the LLM response has completed (not currently generating), - # otherwise defer execution until push_aggregation() is called - # (triggered by LLMFullResponseEndFrame or interruption). - if not self._started: - self._flush_function_call_messages_to_context() - if run_llm: - await self.push_context_frame(FrameDirection.UPSTREAM) + if run_llm: + await self.push_context_frame(FrameDirection.UPSTREAM) + + # Call the `on_context_updated` callback once the function call result + # is added to the context. Also, run this in a separate task to make + # sure we don't block the pipeline. + if properties and properties.on_context_updated: + task_name = f"{frame.function_name}:{frame.tool_call_id}:on_context_updated" + task = self.create_task(properties.on_context_updated(), task_name) + self._context_updated_tasks.add(task) + task.add_done_callback(self._context_updated_task_finished) async def _handle_function_call_cancel(self, frame: FunctionCallCancelFrame): logger.debug( @@ -802,12 +778,7 @@ class LLMAssistantAggregator(LLMContextAggregator): del self._function_calls_in_progress[frame.tool_call_id] def _update_function_call_result(self, function_name: str, tool_call_id: str, result: Any): - def iter_all(): - yield from self._function_calls_context_messages - # In case on long-running function call, the function may already be added to the context - yield from self._context.get_messages() - - for message in iter_all(): + for message in self._context.get_messages(): if ( not isinstance(message, LLMSpecificMessage) and message["role"] == "tool" @@ -822,7 +793,7 @@ class LLMAssistantAggregator(LLMContextAggregator): logger.debug(f"{self} Appending UserImageRawFrame to LLM context (size: {frame.size})") - self._context.add_image_frame_message( + await self._context.add_image_frame_message( format=frame.format, size=frame.size, image=frame.image, diff --git a/src/pipecat/services/hume/tts.py b/src/pipecat/services/hume/tts.py index 673b41b7c..999cd460c 100644 --- a/src/pipecat/services/hume/tts.py +++ b/src/pipecat/services/hume/tts.py @@ -14,12 +14,14 @@ from pydantic import BaseModel from pipecat.frames.frames import ( ErrorFrame, Frame, + InterruptionFrame, StartFrame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, ) -from pipecat.services.tts_service import TTSService +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.tts_service import WordTTSService from pipecat.utils.tracing.service_decorators import traced_tts try: @@ -29,6 +31,7 @@ try: PostedUtterance, PostedUtteranceVoiceWithId, ) + from hume.tts.types import TimestampMessage except ModuleNotFoundError as e: # pragma: no cover - import-time guidance logger.error(f"Exception: {e}") logger.error("In order to use Hume, you need to `pip install pipecat-ai[hume]`.") @@ -38,7 +41,7 @@ except ModuleNotFoundError as e: # pragma: no cover - import-time guidance HUME_SAMPLE_RATE = 48_000 # Hume TTS streams at 48 kHz -class HumeTTSService(TTSService): +class HumeTTSService(WordTTSService): """Hume Octave Text-to-Speech service. Streams PCM audio via Hume's HTTP output streaming (JSON chunks) endpoint @@ -48,6 +51,7 @@ class HumeTTSService(TTSService): - Generates speech from text using Hume TTS. - Streams PCM audio. + - Supports word-level timestamps for precise audio-text synchronization. - Supports dynamic updates of voice and synthesis parameters at runtime. - Provides metrics for Time To First Byte (TTFB) and TTS usage. """ @@ -92,7 +96,13 @@ class HumeTTSService(TTSService): f"Hume TTS streams at {HUME_SAMPLE_RATE} Hz; configured sample_rate={sample_rate}" ) - super().__init__(sample_rate=sample_rate, **kwargs) + # WordTTSService sets push_text_frames=False by default, which we want + super().__init__( + sample_rate=sample_rate, + push_text_frames=False, + push_stop_frames=True, + **kwargs, + ) self._client = AsyncHumeClient(api_key=api_key) self._params = params or HumeTTSService.InputParams() @@ -102,6 +112,10 @@ class HumeTTSService(TTSService): self._audio_bytes = b"" + # Track cumulative time for word timestamps across utterances + self._cumulative_time = 0.0 + self._started = False + def can_generate_metrics(self) -> bool: """Can generate metrics. @@ -117,6 +131,27 @@ class HumeTTSService(TTSService): frame: The start frame. """ await super().start(frame) + self._reset_state() + + def _reset_state(self): + """Reset internal state variables.""" + self._cumulative_time = 0.0 + self._started = False + + async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): + """Push a frame and handle state changes. + + Args: + frame: The frame to push. + direction: The direction to push the frame. + """ + await super().push_frame(frame, direction) + if isinstance(frame, (InterruptionFrame, TTSStoppedFrame)): + # Reset timing on interruption or stop + self._reset_state() + + if isinstance(frame, TTSStoppedFrame): + await self.add_word_timestamps([("Reset", 0)]) async def update_setting(self, key: str, value: Any) -> None: """Runtime updates via `TTSUpdateSettingsFrame`. @@ -133,7 +168,7 @@ class HumeTTSService(TTSService): if key_l == "voice_id": self.set_voice(str(value)) - logger.info(f"HumeTTSService voice_id set to: {self.voice}") + logger.debug(f"HumeTTSService voice_id set to: {self.voice}") elif key_l == "description": self._params.description = None if value is None else str(value) elif key_l == "speed": @@ -146,7 +181,7 @@ class HumeTTSService(TTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - """Generate speech from text using Hume TTS. + """Generate speech from text using Hume TTS with word timestamps. Args: text: The text to be synthesized. @@ -177,7 +212,12 @@ class HumeTTSService(TTSService): await self.start_ttfb_metrics() await self.start_tts_usage_metrics(text) - yield TTSStartedFrame() + + # Start TTS sequence if not already started + if not self._started: + self.start_word_timestamps() + yield TTSStartedFrame() + self._started = True try: # Instant mode is always enabled here (not user-configurable) @@ -188,23 +228,50 @@ class HumeTTSService(TTSService): # Use version "2" by default if no description is provided # Version "1" is needed when description is used version = "1" if self._params.description is not None else "2" + + # Track the duration of this utterance based on the last timestamp + utterance_duration = 0.0 + async for chunk in self._client.tts.synthesize_json_streaming( utterances=[utterance], format=pcm_fmt, instant_mode=True, version=version, + include_timestamp_types=["word"], # Request word-level timestamps ): + # Process audio chunks audio_b64 = getattr(chunk, "audio", None) - if not audio_b64: - continue + if audio_b64: + await self.stop_ttfb_metrics() + pcm_bytes = base64.b64decode(audio_b64) + self._audio_bytes += pcm_bytes - pcm_bytes = base64.b64decode(audio_b64) - self._audio_bytes += pcm_bytes + # Buffer audio until we have enough to avoid glitches + if len(self._audio_bytes) >= self.chunk_size: + frame = TTSAudioRawFrame( + audio=self._audio_bytes, + sample_rate=self.sample_rate, + num_channels=1, + ) + yield frame + self._audio_bytes = b"" - # Buffer audio until we have enough to avoid glitches - if len(self._audio_bytes) < self.chunk_size: - continue + # Process timestamp messages + if isinstance(chunk, TimestampMessage): + timestamp = chunk.timestamp + if timestamp.type == "word": + # Convert milliseconds to seconds and add cumulative offset + word_start_time = self._cumulative_time + (timestamp.time.begin / 1000.0) + word_end_time = self._cumulative_time + (timestamp.time.end / 1000.0) + # Track the maximum end time for this utterance + utterance_duration = max(utterance_duration, word_end_time) + + # Add word timestamp + await self.add_word_timestamps([(timestamp.text, word_start_time)]) + + # Flush any remaining audio bytes + if self._audio_bytes: frame = TTSAudioRawFrame( audio=self._audio_bytes, sample_rate=self.sample_rate, @@ -215,9 +282,13 @@ class HumeTTSService(TTSService): self._audio_bytes = b"" + # Update cumulative time for next utterance + if utterance_duration > 0: + self._cumulative_time = utterance_duration + except Exception as e: await self.push_error(exception=e) finally: # Ensure TTFB timer is stopped even on early failures await self.stop_ttfb_metrics() - yield TTSStoppedFrame() + # Let the parent class handle TTSStoppedFrame via push_stop_frames diff --git a/src/pipecat/services/simli/video.py b/src/pipecat/services/simli/video.py index bac54f35b..e6514ca7b 100644 --- a/src/pipecat/services/simli/video.py +++ b/src/pipecat/services/simli/video.py @@ -84,6 +84,10 @@ class SimliVideoService(FrameProcessor): Please use 'api_key' and 'face_id' parameters instead. use_turn_server: Whether to use TURN server for connection. Defaults to False. + + .. deprecated:: 0.0.95 + The 'use_turn_server' parameter is deprecated and will be removed in a future version. + latency_interval: Latency interval setting for sending health checks to check the latency to Simli Servers. Defaults to 0. simli_url: URL of the simli servers. Can be changed for custom deployments @@ -135,14 +139,20 @@ class SimliVideoService(FrameProcessor): config = SimliConfig(**config_kwargs) + if use_turn_server: + warnings.warn( + "The 'use_turn_server' parameter is deprecated and will be removed in a future version.", + DeprecationWarning, + stacklevel=2, + ) + self._initialized = False # Add buffer time to session limits config.maxIdleTime += 5 config.maxSessionLength += 5 self._simli_client = SimliClient( - config, - use_turn_server, - latency_interval, + config=config, + latencyInterval=latency_interval, simliURL=simli_url, ) diff --git a/uv.lock b/uv.lock index eabe03714..6dec2d8dc 100644 --- a/uv.lock +++ b/uv.lock @@ -4727,7 +4727,7 @@ requires-dist = [ { name = "resampy", specifier = "~=0.4.3" }, { name = "sarvamai", marker = "extra == 'sarvam'", specifier = "==0.1.21" }, { name = "sentry-sdk", marker = "extra == 'sentry'", specifier = ">=2.28.0,<3" }, - { name = "simli-ai", marker = "extra == 'simli'", specifier = "~=0.1.25" }, + { name = "simli-ai", marker = "extra == 'simli'", specifier = "~=1.0.3" }, { name = "soundfile", marker = "extra == 'soundfile'", specifier = "~=0.13.1" }, { name = "soxr", specifier = "~=0.5.0" }, { name = "speechmatics-rt", marker = "extra == 'speechmatics'", specifier = ">=0.5.0" }, @@ -6496,18 +6496,19 @@ wheels = [ [[package]] name = "simli-ai" -version = "0.1.25" +version = "1.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiortc" }, { name = "av" }, { name = "httpx" }, + { name = "livekit" }, { name = "numpy" }, { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/64/6a/b28f90baf76f6a60865985f6233ff44abc72d45b66b76658bff3961e20a7/simli_ai-0.1.25.tar.gz", hash = "sha256:7a00b3426dc26a6a421641072c3e49014b7950c621cf4544152f35c58d13fcff", size = 13182, upload-time = "2025-11-06T16:27:08.862Z" } +sdist = { url = "https://files.pythonhosted.org/packages/81/03/b0b3e12c68fd3f9c57f6afeee67841349e4866b88760f413357af3043ae4/simli_ai-1.0.3.tar.gz", hash = "sha256:e96b0621a1dbd9582b2ae3d51eefd4995983b49c1f1061eb9239707b15a1ee27", size = 13350, upload-time = "2025-11-13T12:22:32.514Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ac/57/ae1032fd88214ea4ee6d3028c817c12a999eb90a67766bbab31e9819385a/simli_ai-0.1.25-py3-none-any.whl", hash = "sha256:7d01f65321dc9052f25e15d0463af6a20a86c6d37d9a7b3a2c4b01cbec0a54ed", size = 13651, upload-time = "2025-11-06T16:27:07.765Z" }, + { url = "https://files.pythonhosted.org/packages/5e/d1/dc382ba529de0d2d51f35e9bfd20b41d8f5c96404a3aa24bae97a5a5e51f/simli_ai-1.0.3-py3-none-any.whl", hash = "sha256:ffafa7540aa28833e207be8f3b199367c7f500dac1a8ba0108395bfb7d8362bc", size = 13863, upload-time = "2025-11-13T12:22:31.218Z" }, ] [[package]]