diff --git a/.claude/skills/cleanup/SKILL.md b/.claude/skills/cleanup/SKILL.md index f7dd6ea98..48c5e0ee8 100644 --- a/.claude/skills/cleanup/SKILL.md +++ b/.claude/skills/cleanup/SKILL.md @@ -293,7 +293,6 @@ class NewTTSService(TTSService): """ super().__init__(**kwargs) self._api_key = api_key - self.set_voice(voice) ``` --- diff --git a/COMMUNITY_INTEGRATIONS.md b/COMMUNITY_INTEGRATIONS.md index a26836a52..642754451 100644 --- a/COMMUNITY_INTEGRATIONS.md +++ b/COMMUNITY_INTEGRATIONS.md @@ -25,7 +25,6 @@ Your repository must contain these components: - **Source code** - Complete implementation following Pipecat patterns - **Foundational example** - Single file example showing basic usage (see [Pipecat examples](https://github.com/pipecat-ai/pipecat/tree/main/examples/foundational)) - **README.md** - Must include: - - Introduction and explanation of your integration - Installation instructions - Usage instructions with Pipecat Pipeline @@ -110,7 +109,6 @@ Once your PR is submitted, post in the `#community-integrations` Discord channel #### Key requirements: - **Frame sequence:** Output must follow this frame sequence pattern: - - `LLMFullResponseStartFrame` - Signals the start of an LLM response - `LLMTextFrame` - Contains LLM content, typically streamed as tokens - `LLMFullResponseEndFrame` - Signals the end of an LLM response @@ -235,22 +233,76 @@ def can_generate_metrics(self) -> bool: ### Dynamic Settings Updates -STT, LLM, and TTS services support `ServiceUpdateSettingsFrame` for dynamic configuration changes. The base STTService has an `_update_settings()` method that handles settings, and the private `_settings` `Dict` is used to store settings and provide access to the subclass. +STT, LLM, and TTS services support runtime configuration changes via `*UpdateSettingsFrame`s (e.g. `STTUpdateSettingsFrame`, `TTSUpdateSettingsFrame`, `LLMUpdateSettingsFrame`). + +Each service declares a settings dataclass that extends the appropriate base (`STTSettings`, `TTSSettings`, `LLMSettings`). Fields default to `NOT_GIVEN` so that update objects can represent sparse deltas: ```python -async def set_language(self, language: Language): - """Set the recognition language and reconnect. +from dataclasses import dataclass, field - Args: - language: The language to use for speech recognition. +from pipecat.services.settings import STTSettings, NOT_GIVEN + +@dataclass +class MySTTSettings(STTSettings): + """Settings for my STT service. + + Parameters: + region: Cloud region for the service. """ - logger.info(f"Switching STT language to: [{language}]") - self._settings["language"] = language - await self._disconnect() - await self._connect() + + region: str = field(default_factory=lambda: NOT_GIVEN) ``` -Note that, in this example, Deepgram requires the websocket connection be disconnected and reconnected to reinitialize the service with the new value. Consider if your service requires reconnection. +The service stores its current settings in `self._settings` and declares the type with a class-level annotation for editor support: + +```python +class MySTTService(STTService): + + _settings: MySTTSettings + + def __init__(self, *, model: str, region: str, **kwargs): + super().__init__(**kwargs) + self._settings = MySTTSettings(model=model, region=region) + self._sync_model_name_to_metrics() +``` + +To react to runtime setting changes, override `_update_settings`. The base implementation applies the delta to `self._settings` and returns a `dict` mapping each changed field name to its **pre-update** value. Your override should call `super()` first, then act on the changed fields. A common implementation might look like: + +```python +async def _update_settings(self, update: STTSettings) -> dict[str, Any]: + """Apply a settings update, reconfiguring the recognizer if needed.""" + changed = await super()._update_settings(update) + + if not changed: + return changed + + await self._disconnect() + await self._connect() + + return changed +``` + +The dict keys work like a set for membership tests (`"language" in changed`) and truthiness (`if changed`). Use `changed.keys() - {"language"}` for set difference, or `changed["language"]` to inspect the previous value of a field. + +Note that, in this example, the service requires a reconnect to apply the new language. Consider, for each setting, whether your service requires reconnection or can apply changes in-place. + +If your service can't yet apply certain settings at runtime, call `self._warn_unhandled_updated_settings(changed)` with any unhandled field names so users get a clear log message: + +```python +async def _update_settings(self, update: STTSettings) -> dict[str, Any]: + changed = await super()._update_settings(update) + + if not changed: + return changed + + if "language" in changed: + await self._update_language() + else: + # TODO: handle changes to other settings soon! + self._warn_unhandled_updated_settings(changed.keys() - {"language"}) + + return changed +``` ### Sample Rate Handling @@ -260,7 +312,7 @@ Sample rates are set via PipelineParams and passed to each frame processor at in async def start(self, frame: StartFrame): """Start the service.""" await super().start(frame) - self._settings["output_format"]["sample_rate"] = self.sample_rate + self._settings.output_sample_rate = self.sample_rate await self._connect() ``` diff --git a/changelog/3714.added.md b/changelog/3714.added.md new file mode 100644 index 000000000..83084675a --- /dev/null +++ b/changelog/3714.added.md @@ -0,0 +1,19 @@ +- Added support for using strongly-typed objects instead of dicts for updating service settings at runtime. + + Instead of, say: + + ```python + await task.queue_frame( + STTUpdateSettingsFrame(settings={"language": Language.ES}) + ) + ``` + + you'd do: + + ```python + await task.queue_frame( + STTUpdateSettingsFrame(update=DeepgramSTTSettings(language=Language.ES)) + ) + ``` + + Each service now vends strongly-typed classes like `DeepgramSTTSettings` representing the service's runtime-updatable settings. diff --git a/changelog/3714.changed.md b/changelog/3714.changed.md new file mode 100644 index 000000000..bcfb5cbf7 --- /dev/null +++ b/changelog/3714.changed.md @@ -0,0 +1 @@ +- ⚠️ Refactored runtime-updatable service settings to use strongly-typed classes (`TTSSettings`, `STTSettings`, `LLMSettings`, and service-specific subclasses) instead of plain dicts. Each service's `_settings` now holds these strongly-typed objects. For service maintainers, see changes in COMMUNITY_INTEGRATIONS.md. diff --git a/changelog/3714.deprecated.2.md b/changelog/3714.deprecated.2.md new file mode 100644 index 000000000..232c1dee5 --- /dev/null +++ b/changelog/3714.deprecated.2.md @@ -0,0 +1 @@ +- Dict-based `*UpdateSettingsFrame(settings={...})` is deprecated in favor of passing typed settings delta objects with `*UpdateSettingsFrame(update={...})`. diff --git a/changelog/3714.deprecated.md b/changelog/3714.deprecated.md new file mode 100644 index 000000000..75337a642 --- /dev/null +++ b/changelog/3714.deprecated.md @@ -0,0 +1,3 @@ +- Deprecated `set_model()`, `set_voice()`, and `set_language()` on AI services in favor of runtime updates via `TTSUpdateSettingsFrame`, `STTUpdateSettingsFrame`, and `LLMUpdateSettingsFrame`. + + ⚠️ Note, too, a subtle behavior change in these deprecated methods. Whereas previously only `set_language()` caused the service to actually react to the update (e.g. by reconnecting to a remote service so it an pick up the change), now all these methods do. This change was made as part of a refactor making them all work the same way under the hood. diff --git a/examples/foundational/14n-function-calling-perplexity.py b/examples/foundational/14n-function-calling-perplexity.py index 40041aa34..2f1a18d52 100644 --- a/examples/foundational/14n-function-calling-perplexity.py +++ b/examples/foundational/14n-function-calling-perplexity.py @@ -65,7 +65,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) - llm = PerplexityLLMService(api_key=os.getenv("PERPLEXITY_API_KEY"), model="sonar") + llm = PerplexityLLMService(api_key=os.getenv("PERPLEXITY_API_KEY")) messages = [ { diff --git a/examples/foundational/14s-function-calling-sambanova.py b/examples/foundational/14s-function-calling-sambanova.py index 79c43a473..76eb390c0 100644 --- a/examples/foundational/14s-function-calling-sambanova.py +++ b/examples/foundational/14s-function-calling-sambanova.py @@ -70,10 +70,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) - llm = SambaNovaLLMService( - api_key=os.getenv("SAMBANOVA_API_KEY"), - model="Llama-4-Maverick-17B-128E-Instruct", - ) + llm = SambaNovaLLMService(api_key=os.getenv("SAMBANOVA_API_KEY")) # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. llm.register_function("get_current_weather", fetch_weather_from_api) diff --git a/examples/foundational/35-pattern-pair-voice-switching.py b/examples/foundational/35-pattern-pair-voice-switching.py index 4b269ac3e..cacc04459 100644 --- a/examples/foundational/35-pattern-pair-voice-switching.py +++ b/examples/foundational/35-pattern-pair-voice-switching.py @@ -117,7 +117,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # First flush any existing audio to finish the current context await tts.flush_audio() # Then set the new voice - tts.set_voice(VOICE_IDS[voice_name]) + await tts.set_voice(VOICE_IDS[voice_name]) logger.info(f"Switched to {voice_name} voice") else: logger.warning(f"Unknown voice: {voice_name}") diff --git a/examples/foundational/55a-update-settings-deepgram-flux-stt.py b/examples/foundational/55a-update-settings-deepgram-flux-stt.py new file mode 100644 index 000000000..d5fb66a2e --- /dev/null +++ b/examples/foundational/55a-update-settings-deepgram-flux-stt.py @@ -0,0 +1,128 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, STTUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.flux.stt import DeepgramFluxSTTService, DeepgramFluxSTTSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramFluxSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Deepgram Flux STT settings: language=es") + await task.queue_frame( + STTUpdateSettingsFrame(update=DeepgramFluxSTTSettings(language=Language.ES)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55a-update-settings-deepgram-sagemaker-stt.py b/examples/foundational/55a-update-settings-deepgram-sagemaker-stt.py new file mode 100644 index 000000000..8e45b5f2a --- /dev/null +++ b/examples/foundational/55a-update-settings-deepgram-sagemaker-stt.py @@ -0,0 +1,134 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, STTUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt_sagemaker import ( + DeepgramSageMakerSTTService, + DeepgramSageMakerSTTSettings, +) +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSageMakerSTTService( + endpoint_name=os.getenv("SAGEMAKER_STT_ENDPOINT_NAME"), + region=os.getenv("AWS_REGION"), + ) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Deepgram SageMaker STT settings: language=es") + await task.queue_frame( + STTUpdateSettingsFrame(update=DeepgramSageMakerSTTSettings(language=Language.ES)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55a-update-settings-deepgram-stt.py b/examples/foundational/55a-update-settings-deepgram-stt.py new file mode 100644 index 000000000..aea9475a8 --- /dev/null +++ b/examples/foundational/55a-update-settings-deepgram-stt.py @@ -0,0 +1,128 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, STTUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService, DeepgramSTTSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Deepgram STT settings: language=es") + await task.queue_frame( + STTUpdateSettingsFrame(update=DeepgramSTTSettings(language=Language.ES)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55b-update-settings-azure-stt.py b/examples/foundational/55b-update-settings-azure-stt.py new file mode 100644 index 000000000..7fd0d2ca4 --- /dev/null +++ b/examples/foundational/55b-update-settings-azure-stt.py @@ -0,0 +1,131 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, STTUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.azure.stt import AzureSTTService, AzureSTTSettings +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = AzureSTTService( + api_key=os.getenv("AZURE_SPEECH_API_KEY"), + region=os.getenv("AZURE_SPEECH_REGION"), + ) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Azure STT settings: language=es") + await task.queue_frame( + STTUpdateSettingsFrame(update=AzureSTTSettings(language=Language.ES)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55c-update-settings-google-stt.py b/examples/foundational/55c-update-settings-google-stt.py new file mode 100644 index 000000000..dd33bfe75 --- /dev/null +++ b/examples/foundational/55c-update-settings-google-stt.py @@ -0,0 +1,128 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, STTUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.google.stt import GoogleSTTService, GoogleSTTSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = GoogleSTTService(credentials=os.getenv("GOOGLE_TEST_CREDENTIALS")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Google STT settings: language=es") + await task.queue_frame( + STTUpdateSettingsFrame(update=GoogleSTTSettings(language=Language.ES)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55d-update-settings-assemblyai-stt.py b/examples/foundational/55d-update-settings-assemblyai-stt.py new file mode 100644 index 000000000..6d6a2532e --- /dev/null +++ b/examples/foundational/55d-update-settings-assemblyai-stt.py @@ -0,0 +1,128 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, STTUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.assemblyai.stt import AssemblyAISTTService, AssemblyAISTTSettings +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = AssemblyAISTTService(api_key=os.getenv("ASSEMBLYAI_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating AssemblyAI STT settings: language=es") + await task.queue_frame( + STTUpdateSettingsFrame(update=AssemblyAISTTSettings(language=Language.ES)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55e-update-settings-gladia-stt.py b/examples/foundational/55e-update-settings-gladia-stt.py new file mode 100644 index 000000000..a2c6f21fe --- /dev/null +++ b/examples/foundational/55e-update-settings-gladia-stt.py @@ -0,0 +1,128 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, STTUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.gladia.stt import GladiaSTTService, GladiaSTTSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = GladiaSTTService(api_key=os.getenv("GLADIA_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Gladia STT settings: language=es") + await task.queue_frame( + STTUpdateSettingsFrame(update=GladiaSTTSettings(language=Language.ES)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55f-update-settings-elevenlabs-realtime-stt.py b/examples/foundational/55f-update-settings-elevenlabs-realtime-stt.py new file mode 100644 index 000000000..9aee04fbb --- /dev/null +++ b/examples/foundational/55f-update-settings-elevenlabs-realtime-stt.py @@ -0,0 +1,131 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, STTUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.elevenlabs.stt import ( + ElevenLabsRealtimeSTTService, + ElevenLabsRealtimeSTTSettings, +) +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = ElevenLabsRealtimeSTTService(api_key=os.getenv("ELEVENLABS_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating ElevenLabs Realtime STT settings: language=es") + await task.queue_frame( + STTUpdateSettingsFrame(update=ElevenLabsRealtimeSTTSettings(language=Language.ES)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55g-update-settings-elevenlabs-stt.py b/examples/foundational/55g-update-settings-elevenlabs-stt.py new file mode 100644 index 000000000..33844935a --- /dev/null +++ b/examples/foundational/55g-update-settings-elevenlabs-stt.py @@ -0,0 +1,133 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +import aiohttp +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, STTUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.elevenlabs.stt import ElevenLabsSTTService, ElevenLabsSTTSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + async with aiohttp.ClientSession() as session: + stt = ElevenLabsSTTService( + api_key=os.getenv("ELEVENLABS_API_KEY"), + aiohttp_session=session, + ) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating ElevenLabs STT settings: language=es") + await task.queue_frame( + STTUpdateSettingsFrame(update=ElevenLabsSTTSettings(language=Language.ES)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55h-update-settings-speechmatics-stt.py b/examples/foundational/55h-update-settings-speechmatics-stt.py new file mode 100644 index 000000000..d041d69d2 --- /dev/null +++ b/examples/foundational/55h-update-settings-speechmatics-stt.py @@ -0,0 +1,153 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, STTUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.services.speechmatics.stt import SpeechmaticsSTTService, SpeechmaticsSTTSettings +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = SpeechmaticsSTTService( + api_key=os.getenv("SPEECHMATICS_API_KEY"), + params=SpeechmaticsSTTService.InputParams( + enable_diarization=True, + speaker_active_format="<{speaker_id}>{text}", + speaker_passive_format="<{speaker_id}>{text}", + ), + ) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Speechmatics STT settings: language=es") + await task.queue_frame( + STTUpdateSettingsFrame(update=SpeechmaticsSTTSettings(language=Language.ES)) + ) + + await asyncio.sleep(10) + logger.info("Updating Speechmatics STT settings: focus_speakers=['S1']") + await task.queue_frame( + STTUpdateSettingsFrame(update=SpeechmaticsSTTSettings(focus_speakers=["S1"])) + ) + + await asyncio.sleep(10) + logger.info( + "Updating Speechmatics STT settings: speaker_active_format={text}" + ) + await task.queue_frame( + STTUpdateSettingsFrame( + update=SpeechmaticsSTTSettings( + speaker_active_format="{text}" + ) + ) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55i-update-settings-whisper-api-stt.py b/examples/foundational/55i-update-settings-whisper-api-stt.py new file mode 100644 index 000000000..1d5022674 --- /dev/null +++ b/examples/foundational/55i-update-settings-whisper-api-stt.py @@ -0,0 +1,132 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, STTUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.services.openai.stt import OpenAISTTService +from pipecat.services.whisper.base_stt import BaseWhisperSTTSettings +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + # This file is meant to exercise Whisper API-based STT services, so we use + # OpenAI's Whisper STT as an example here. Here we could've also used: + # - SambaNova + # - Groq + stt = OpenAISTTService( + api_key=os.getenv("OPENAI_API_KEY"), + ) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info('Updating OpenAI STT settings: language="es"') + await task.queue_frame(STTUpdateSettingsFrame(update=BaseWhisperSTTSettings(language="es"))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55j-update-settings-sarvam-stt.py b/examples/foundational/55j-update-settings-sarvam-stt.py new file mode 100644 index 000000000..e39c5cb5a --- /dev/null +++ b/examples/foundational/55j-update-settings-sarvam-stt.py @@ -0,0 +1,128 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, STTUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.services.sarvam.stt import SarvamSTTService, SarvamSTTSettings +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = SarvamSTTService(api_key=os.getenv("SARVAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Sarvam STT settings: language=en-IN") + await task.queue_frame( + STTUpdateSettingsFrame(update=SarvamSTTSettings(language=Language.EN_IN)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55k-update-settings-soniox-stt.py b/examples/foundational/55k-update-settings-soniox-stt.py new file mode 100644 index 000000000..2cbcd44f4 --- /dev/null +++ b/examples/foundational/55k-update-settings-soniox-stt.py @@ -0,0 +1,128 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, STTUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.services.soniox.stt import SonioxSTTService, SonioxSTTSettings +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = SonioxSTTService(api_key=os.getenv("SONIOX_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Soniox STT settings: language=es") + await task.queue_frame( + STTUpdateSettingsFrame(update=SonioxSTTSettings(language=Language.ES)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55l-update-settings-aws-transcribe-stt.py b/examples/foundational/55l-update-settings-aws-transcribe-stt.py new file mode 100644 index 000000000..0f4c18981 --- /dev/null +++ b/examples/foundational/55l-update-settings-aws-transcribe-stt.py @@ -0,0 +1,128 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, STTUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.aws.stt import AWSTranscribeSTTService, AWSTranscribeSTTSettings +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = AWSTranscribeSTTService() + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating AWS Transcribe STT settings: language=es") + await task.queue_frame( + STTUpdateSettingsFrame(update=AWSTranscribeSTTSettings(language=Language.ES)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55m-update-settings-cartesia-stt.py b/examples/foundational/55m-update-settings-cartesia-stt.py new file mode 100644 index 000000000..6ba27a85e --- /dev/null +++ b/examples/foundational/55m-update-settings-cartesia-stt.py @@ -0,0 +1,128 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, STTUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.stt import CartesiaSTTService, CartesiaSTTSettings +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = CartesiaSTTService(api_key=os.getenv("CARTESIA_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Cartesia STT settings: language=es") + await task.queue_frame( + STTUpdateSettingsFrame(update=CartesiaSTTSettings(language=Language.ES)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55n-update-settings-cartesia-http-tts.py b/examples/foundational/55n-update-settings-cartesia-http-tts.py new file mode 100644 index 000000000..27cee5b8f --- /dev/null +++ b/examples/foundational/55n-update-settings-cartesia-http-tts.py @@ -0,0 +1,133 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import ( + CartesiaHttpTTSService, + CartesiaTTSSettings, + GenerationConfig, +) +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaHttpTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Cartesia HTTP TTS settings: speed increased to 1.5") + await task.queue_frame( + TTSUpdateSettingsFrame( + update=CartesiaTTSSettings(generation_config=GenerationConfig(speed=1.5)) + ) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55n-update-settings-cartesia-tts.py b/examples/foundational/55n-update-settings-cartesia-tts.py new file mode 100644 index 000000000..303c23a25 --- /dev/null +++ b/examples/foundational/55n-update-settings-cartesia-tts.py @@ -0,0 +1,132 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService, CartesiaTTSSettings, GenerationConfig +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +# We use lambdas to defer transport parameter creation until the transport +# type is selected at runtime. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + user_aggregator, # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + assistant_aggregator, # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Cartesia TTS settings: speed increased to 1.5") + await task.queue_frame( + TTSUpdateSettingsFrame( + update=CartesiaTTSSettings(generation_config=GenerationConfig(speed=1.5)) + ) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55o-update-settings-elevenlabs-http-tts.py b/examples/foundational/55o-update-settings-elevenlabs-http-tts.py new file mode 100644 index 000000000..a67202702 --- /dev/null +++ b/examples/foundational/55o-update-settings-elevenlabs-http-tts.py @@ -0,0 +1,132 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +import asyncio +import os + +import aiohttp +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.elevenlabs.tts import ElevenLabsHttpTTSService, ElevenLabsHttpTTSSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + async with aiohttp.ClientSession() as session: + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = ElevenLabsHttpTTSService( + api_key=os.getenv("ELEVENLABS_API_KEY"), + voice_id=os.getenv("ELEVENLABS_VOICE_ID"), + aiohttp_session=session, + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating ElevenLabs TTS settings: speed=0.7") + await task.queue_frame( + TTSUpdateSettingsFrame(update=ElevenLabsHttpTTSSettings(speed=0.7)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55o-update-settings-elevenlabs-tts.py b/examples/foundational/55o-update-settings-elevenlabs-tts.py new file mode 100644 index 000000000..3fefa1ffb --- /dev/null +++ b/examples/foundational/55o-update-settings-elevenlabs-tts.py @@ -0,0 +1,134 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.elevenlabs.tts import ElevenLabsTTSService, ElevenLabsTTSSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = ElevenLabsTTSService( + api_key=os.getenv("ELEVENLABS_API_KEY"), + voice_id=os.getenv("ELEVENLABS_VOICE_ID"), + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating ElevenLabs TTS settings: speed=0.7") + await task.queue_frame(TTSUpdateSettingsFrame(update=ElevenLabsTTSSettings(speed=0.7))) + + await asyncio.sleep(10) + logger.info("Updating ElevenLabs TTS settings: switching to a different voice") + await task.queue_frame( + TTSUpdateSettingsFrame( + update=ElevenLabsTTSSettings(voice=os.getenv("ELEVENLABS_VOICE_ID_ALT")) + ) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55p-update-settings-openai-tts.py b/examples/foundational/55p-update-settings-openai-tts.py new file mode 100644 index 000000000..5aef081fc --- /dev/null +++ b/examples/foundational/55p-update-settings-openai-tts.py @@ -0,0 +1,123 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.services.openai.tts import OpenAITTSService, OpenAITTSSettings +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = OpenAITTSService(api_key=os.getenv("OPENAI_API_KEY")) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + audio_out_sample_rate=24000, + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating OpenAI TTS settings: speed=2.0") + await task.queue_frame(TTSUpdateSettingsFrame(update=OpenAITTSSettings(speed=2.0))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55q-update-settings-deepgram-http-tts.py b/examples/foundational/55q-update-settings-deepgram-http-tts.py new file mode 100644 index 000000000..64bbea587 --- /dev/null +++ b/examples/foundational/55q-update-settings-deepgram-http-tts.py @@ -0,0 +1,137 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +import asyncio +import os + +import aiohttp +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.deepgram.tts import DeepgramHttpTTSService, DeepgramTTSSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + async with aiohttp.ClientSession() as session: + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = DeepgramHttpTTSService( + api_key=os.getenv("DEEPGRAM_API_KEY"), + aiohttp_session=session, + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info('Updating Deepgram TTS settings: voice="aura-2-aries-en"') + await task.queue_frame( + TTSUpdateSettingsFrame(update=DeepgramTTSSettings(voice="aura-2-aries-en")) + ) + + await asyncio.sleep(10) + logger.info('Updating Deepgram TTS settings: voice="aura-2-luna-en"') + await task.queue_frame( + TTSUpdateSettingsFrame(update=DeepgramTTSSettings(voice="aura-2-luna-en")) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55q-update-settings-deepgram-sagemaker-tts.py b/examples/foundational/55q-update-settings-deepgram-sagemaker-tts.py new file mode 100644 index 000000000..35fb7cebe --- /dev/null +++ b/examples/foundational/55q-update-settings-deepgram-sagemaker-tts.py @@ -0,0 +1,137 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.deepgram.tts_sagemaker import ( + DeepgramSageMakerTTSService, + DeepgramSageMakerTTSSettings, +) +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = DeepgramSageMakerTTSService( + endpoint_name=os.getenv("SAGEMAKER_TTS_ENDPOINT_NAME"), + region=os.getenv("AWS_REGION"), + voice="aura-2-helena-en", + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info('Updating Deepgram SageMaker TTS settings: voice="aura-2-aries-en"') + await task.queue_frame( + TTSUpdateSettingsFrame(update=DeepgramSageMakerTTSSettings(voice="aura-2-aries-en")) + ) + + await asyncio.sleep(10) + logger.info('Updating Deepgram SageMaker TTS settings: voice="aura-2-luna-en"') + await task.queue_frame( + TTSUpdateSettingsFrame(update=DeepgramSageMakerTTSSettings(voice="aura-2-luna-en")) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55q-update-settings-deepgram-tts.py b/examples/foundational/55q-update-settings-deepgram-tts.py new file mode 100644 index 000000000..9d94a50da --- /dev/null +++ b/examples/foundational/55q-update-settings-deepgram-tts.py @@ -0,0 +1,130 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.deepgram.tts import DeepgramTTSService, DeepgramTTSSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = DeepgramTTSService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info('Updating Deepgram TTS settings: voice="aura-2-aries-en"') + await task.queue_frame( + TTSUpdateSettingsFrame(update=DeepgramTTSSettings(voice="aura-2-aries-en")) + ) + + await asyncio.sleep(10) + logger.info('Updating Deepgram TTS settings: voice="aura-2-luna-en"') + await task.queue_frame( + TTSUpdateSettingsFrame(update=DeepgramTTSSettings(voice="aura-2-luna-en")) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55r-update-settings-azure-http-tts.py b/examples/foundational/55r-update-settings-azure-http-tts.py new file mode 100644 index 000000000..3132580ed --- /dev/null +++ b/examples/foundational/55r-update-settings-azure-http-tts.py @@ -0,0 +1,127 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.azure.tts import AzureHttpTTSService, AzureTTSSettings +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = AzureHttpTTSService( + api_key=os.getenv("AZURE_SPEECH_API_KEY"), + region=os.getenv("AZURE_SPEECH_REGION"), + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info('Updating Azure TTS settings: rate="0.7", style="sad"') + await task.queue_frame( + TTSUpdateSettingsFrame(update=AzureTTSSettings(rate="0.7", style="sad")) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55r-update-settings-azure-tts.py b/examples/foundational/55r-update-settings-azure-tts.py new file mode 100644 index 000000000..d156eab43 --- /dev/null +++ b/examples/foundational/55r-update-settings-azure-tts.py @@ -0,0 +1,127 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.azure.tts import AzureTTSService, AzureTTSSettings +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = AzureTTSService( + api_key=os.getenv("AZURE_SPEECH_API_KEY"), + region=os.getenv("AZURE_SPEECH_REGION"), + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info('Updating Azure TTS settings: rate="0.7", style="sad"') + await task.queue_frame( + TTSUpdateSettingsFrame(update=AzureTTSSettings(rate="0.7", style="sad")) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55s-update-settings-google-http-tts.py b/examples/foundational/55s-update-settings-google-http-tts.py new file mode 100644 index 000000000..6c302411a --- /dev/null +++ b/examples/foundational/55s-update-settings-google-http-tts.py @@ -0,0 +1,124 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.google.tts import GoogleHttpTTSService, GoogleHttpTTSSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = GoogleHttpTTSService(credentials=os.getenv("GOOGLE_TEST_CREDENTIALS")) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Google HTTP TTS settings: speaking_rate=1.4") + await task.queue_frame( + TTSUpdateSettingsFrame(update=GoogleHttpTTSSettings(speaking_rate=1.4)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55s-update-settings-google-stream-tts.py b/examples/foundational/55s-update-settings-google-stream-tts.py new file mode 100644 index 000000000..42e07c64b --- /dev/null +++ b/examples/foundational/55s-update-settings-google-stream-tts.py @@ -0,0 +1,124 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.google.tts import GoogleStreamTTSSettings, GoogleTTSService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = GoogleTTSService(credentials=os.getenv("GOOGLE_TEST_CREDENTIALS")) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Google Stream TTS settings: speaking_rate=1.4") + await task.queue_frame( + TTSUpdateSettingsFrame(update=GoogleStreamTTSSettings(speaking_rate=1.4)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55t-update-settings-playht-tts.py b/examples/foundational/55t-update-settings-playht-tts.py new file mode 100644 index 000000000..ec468a81c --- /dev/null +++ b/examples/foundational/55t-update-settings-playht-tts.py @@ -0,0 +1,126 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.services.playht.tts import PlayHTTTSService, PlayHTTTSSettings +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = PlayHTTTSService( + api_key=os.getenv("PLAYHT_API_KEY"), + user_id=os.getenv("PLAYHT_USER_ID"), + voice_url=os.getenv("PLAYHT_VOICE_URL", ""), + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating PlayHT TTS settings: speed=1.3") + await task.queue_frame(TTSUpdateSettingsFrame(update=PlayHTTTSSettings(speed=1.3))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55u-update-settings-rime-http-tts.py b/examples/foundational/55u-update-settings-rime-http-tts.py new file mode 100644 index 000000000..7b1c9b0fe --- /dev/null +++ b/examples/foundational/55u-update-settings-rime-http-tts.py @@ -0,0 +1,128 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +import asyncio +import os + +import aiohttp +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.services.rime.tts import RimeHttpTTSService, RimeTTSSettings +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + async with aiohttp.ClientSession() as session: + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = RimeHttpTTSService( + api_key=os.getenv("RIME_API_KEY"), voice_id="eva", aiohttp_session=session + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Rime TTS settings: voice=rex") + await task.queue_frame(TTSUpdateSettingsFrame(update=RimeTTSSettings(voice="rex"))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55u-update-settings-rime-tts.py b/examples/foundational/55u-update-settings-rime-tts.py new file mode 100644 index 000000000..0704645f5 --- /dev/null +++ b/examples/foundational/55u-update-settings-rime-tts.py @@ -0,0 +1,125 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.services.rime.tts import RimeTTSService, RimeTTSSettings +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = RimeTTSService( + api_key=os.getenv("RIME_API_KEY"), + voice_id="luna", + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Rime TTS settings: voice=bond") + await task.queue_frame(TTSUpdateSettingsFrame(update=RimeTTSSettings(voice="bond"))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55v-update-settings-lmnt-tts.py b/examples/foundational/55v-update-settings-lmnt-tts.py new file mode 100644 index 000000000..d98462e20 --- /dev/null +++ b/examples/foundational/55v-update-settings-lmnt-tts.py @@ -0,0 +1,125 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.lmnt.tts import LmntTTSService, LmntTTSSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = LmntTTSService( + api_key=os.getenv("LMNT_API_KEY"), + voice_id="lily", + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info('Updating LMNT TTS settings: voice="tyler"') + await task.queue_frame(TTSUpdateSettingsFrame(update=LmntTTSSettings(voice="tyler"))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55w-update-settings-fish-tts.py b/examples/foundational/55w-update-settings-fish-tts.py new file mode 100644 index 000000000..82722ec34 --- /dev/null +++ b/examples/foundational/55w-update-settings-fish-tts.py @@ -0,0 +1,127 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.fish.tts import FishAudioTTSService, FishAudioTTSSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = FishAudioTTSService( + api_key=os.getenv("FISH_API_KEY"), + model="4ce7e917cedd4bc2bb2e6ff3a46acaa1", # Barack Obama + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Fish Audio TTS settings: prosody_speed=1.5") + await task.queue_frame( + TTSUpdateSettingsFrame(update=FishAudioTTSSettings(prosody_speed=1.5)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55x-update-settings-minimax-tts.py b/examples/foundational/55x-update-settings-minimax-tts.py new file mode 100644 index 000000000..306b8f2bd --- /dev/null +++ b/examples/foundational/55x-update-settings-minimax-tts.py @@ -0,0 +1,130 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +import aiohttp +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.minimax.tts import MiniMaxHttpTTSService, MiniMaxTTSSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + async with aiohttp.ClientSession() as session: + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = MiniMaxHttpTTSService( + api_key=os.getenv("MINIMAX_API_KEY", ""), + group_id=os.getenv("MINIMAX_GROUP_ID", ""), + aiohttp_session=session, + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info('Updating MiniMax TTS settings: speed=1.5, emotion="happy"') + await task.queue_frame( + TTSUpdateSettingsFrame(update=MiniMaxTTSSettings(speed=1.5, emotion="happy")) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55y-update-settings-groq-tts.py b/examples/foundational/55y-update-settings-groq-tts.py new file mode 100644 index 000000000..e6ce851c6 --- /dev/null +++ b/examples/foundational/55y-update-settings-groq-tts.py @@ -0,0 +1,122 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.groq.tts import GroqTTSService, GroqTTSSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = GroqTTSService(api_key=os.getenv("GROQ_API_KEY")) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Groq TTS settings: speed=1.5") + await task.queue_frame(TTSUpdateSettingsFrame(update=GroqTTSSettings(speed=1.5))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55z-update-settings-hume-tts.py b/examples/foundational/55z-update-settings-hume-tts.py new file mode 100644 index 000000000..427b99bab --- /dev/null +++ b/examples/foundational/55z-update-settings-hume-tts.py @@ -0,0 +1,129 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.hume.tts import HumeTTSService, HumeTTSSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = HumeTTSService( + api_key=os.getenv("HUME_API_KEY"), + voice_id="f898a92e-685f-43fa-985b-a46920f0650b", + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info('Updating Hume TTS settings: speed=2.0, description="Speak with excitement"') + await task.queue_frame( + TTSUpdateSettingsFrame( + update=HumeTTSSettings(speed=2.0, description="Speak with excitement") + ) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55za-update-settings-neuphonic-http-tts.py b/examples/foundational/55za-update-settings-neuphonic-http-tts.py new file mode 100644 index 000000000..056b32349 --- /dev/null +++ b/examples/foundational/55za-update-settings-neuphonic-http-tts.py @@ -0,0 +1,127 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +import aiohttp +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.neuphonic.tts import NeuphonicHttpTTSService, NeuphonicTTSSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + async with aiohttp.ClientSession() as session: + tts = NeuphonicHttpTTSService( + api_key=os.getenv("NEUPHONIC_API_KEY"), + aiohttp_session=session, + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Neuphonic HTTP TTS settings: speed=1.4") + await task.queue_frame(TTSUpdateSettingsFrame(update=NeuphonicTTSSettings(speed=1.4))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55za-update-settings-neuphonic-tts.py b/examples/foundational/55za-update-settings-neuphonic-tts.py new file mode 100644 index 000000000..187594c7e --- /dev/null +++ b/examples/foundational/55za-update-settings-neuphonic-tts.py @@ -0,0 +1,122 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.neuphonic.tts import NeuphonicTTSService, NeuphonicTTSSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = NeuphonicTTSService(api_key=os.getenv("NEUPHONIC_API_KEY")) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Neuphonic TTS settings: speed=1.4") + await task.queue_frame(TTSUpdateSettingsFrame(update=NeuphonicTTSSettings(speed=1.4))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zb-update-settings-inworld-http-tts.py b/examples/foundational/55zb-update-settings-inworld-http-tts.py new file mode 100644 index 000000000..933a27013 --- /dev/null +++ b/examples/foundational/55zb-update-settings-inworld-http-tts.py @@ -0,0 +1,130 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +import asyncio +import os + +import aiohttp +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.inworld.tts import InworldHttpTTSService, InworldTTSSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + async with aiohttp.ClientSession() as session: + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = InworldHttpTTSService(api_key=os.getenv("INWORLD_API_KEY"), aiohttp_session=session) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Inworld TTS settings: speaking_rate=1.5, temperature=0.8") + await task.queue_frame( + TTSUpdateSettingsFrame( + update=InworldTTSSettings(speaking_rate=1.5, temperature=0.8) + ) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zb-update-settings-inworld-tts.py b/examples/foundational/55zb-update-settings-inworld-tts.py new file mode 100644 index 000000000..f8a66bdd8 --- /dev/null +++ b/examples/foundational/55zb-update-settings-inworld-tts.py @@ -0,0 +1,124 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.inworld.tts import InworldTTSService, InworldTTSSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = InworldTTSService(api_key=os.getenv("INWORLD_API_KEY")) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Inworld TTS settings: speaking_rate=1.5, temperature=0.8") + await task.queue_frame( + TTSUpdateSettingsFrame(update=InworldTTSSettings(speaking_rate=1.5, temperature=0.8)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zc-update-settings-gemini-tts.py b/examples/foundational/55zc-update-settings-gemini-tts.py new file mode 100644 index 000000000..6af28e69f --- /dev/null +++ b/examples/foundational/55zc-update-settings-gemini-tts.py @@ -0,0 +1,133 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.google.tts import GeminiTTSService, GeminiTTSSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = GeminiTTSService( + credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + model="gemini-2.5-flash-tts", + voice_id="Charon", + params=GeminiTTSService.InputParams( + language=Language.EN_US, + prompt="You are a helpful AI assistant. Speak in a natural, conversational tone.", + ), + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info('Updating Gemini TTS settings: prompt="Speak slowly and dramatically"') + await task.queue_frame( + TTSUpdateSettingsFrame(update=GeminiTTSSettings(prompt="Speak slowly and dramatically")) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zd-update-settings-aws-polly-tts.py b/examples/foundational/55zd-update-settings-aws-polly-tts.py new file mode 100644 index 000000000..3d9f72cf4 --- /dev/null +++ b/examples/foundational/55zd-update-settings-aws-polly-tts.py @@ -0,0 +1,122 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.aws.tts import AWSPollyTTSService, AWSPollyTTSSettings +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = AWSPollyTTSService() + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info('Updating AWS Polly TTS settings: rate="fast"') + await task.queue_frame(TTSUpdateSettingsFrame(update=AWSPollyTTSSettings(rate="fast"))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55ze-update-settings-sarvam-http-tts.py b/examples/foundational/55ze-update-settings-sarvam-http-tts.py new file mode 100644 index 000000000..0afce361a --- /dev/null +++ b/examples/foundational/55ze-update-settings-sarvam-http-tts.py @@ -0,0 +1,126 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +import asyncio +import os + +import aiohttp +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.services.sarvam.tts import SarvamHttpTTSService, SarvamHttpTTSSettings +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + async with aiohttp.ClientSession() as session: + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = SarvamHttpTTSService(api_key=os.getenv("SARVAM_API_KEY"), aiohttp_session=session) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Sarvam TTS settings: pace=1.5") + await task.queue_frame(TTSUpdateSettingsFrame(update=SarvamHttpTTSSettings(pace=1.5))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55ze-update-settings-sarvam-tts.py b/examples/foundational/55ze-update-settings-sarvam-tts.py new file mode 100644 index 000000000..98408c4b8 --- /dev/null +++ b/examples/foundational/55ze-update-settings-sarvam-tts.py @@ -0,0 +1,122 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.services.sarvam.tts import SarvamTTSService, SarvamTTSSettings +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = SarvamTTSService(api_key=os.getenv("SARVAM_API_KEY")) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Sarvam TTS settings: pace=1.5") + await task.queue_frame(TTSUpdateSettingsFrame(update=SarvamTTSSettings(pace=1.5))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zf-update-settings-camb-tts.py b/examples/foundational/55zf-update-settings-camb-tts.py new file mode 100644 index 000000000..1fe758849 --- /dev/null +++ b/examples/foundational/55zf-update-settings-camb-tts.py @@ -0,0 +1,123 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.camb.tts import CambTTSService, CambTTSSettings +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CambTTSService(api_key=os.getenv("CAMB_API_KEY")) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Camb TTS settings: language -> Spanish") + await task.queue_frame(TTSUpdateSettingsFrame(update=CambTTSSettings(language=Language.ES))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zg-update-settings-hathora-tts.py b/examples/foundational/55zg-update-settings-hathora-tts.py new file mode 100644 index 000000000..363ac7d85 --- /dev/null +++ b/examples/foundational/55zg-update-settings-hathora-tts.py @@ -0,0 +1,125 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.hathora.tts import HathoraTTSService, HathoraTTSSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = HathoraTTSService( + api_key=os.getenv("HATHORA_API_KEY"), + model="hathora-ai/polar", + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Hathora TTS settings: speed=1.3") + await task.queue_frame(TTSUpdateSettingsFrame(update=HathoraTTSSettings(speed=1.3))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zh-update-settings-resembleai-tts.py b/examples/foundational/55zh-update-settings-resembleai-tts.py new file mode 100644 index 000000000..39b745500 --- /dev/null +++ b/examples/foundational/55zh-update-settings-resembleai-tts.py @@ -0,0 +1,129 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.services.resembleai.tts import ResembleAITTSService, ResembleAITTSSettings +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = ResembleAITTSService( + api_key=os.getenv("RESEMBLE_API_KEY"), + voice_id=os.getenv("RESEMBLE_VOICE_UUID"), + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating ResembleAI TTS settings: voice (changed)") + await task.queue_frame( + TTSUpdateSettingsFrame( + update=ResembleAITTSSettings(voice=os.getenv("RESEMBLE_VOICE_UUID_ALT")) + ) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zi-update-settings-azure-llm.py b/examples/foundational/55zi-update-settings-azure-llm.py new file mode 100644 index 000000000..94cb723e3 --- /dev/null +++ b/examples/foundational/55zi-update-settings-azure-llm.py @@ -0,0 +1,130 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.azure.llm import AzureLLMService +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.base_llm import OpenAILLMSettings +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = AzureLLMService( + api_key=os.getenv("AZURE_CHATGPT_API_KEY"), + endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + model=os.getenv("AZURE_CHATGPT_MODEL"), + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Azure LLM settings: temperature=0.1") + await task.queue_frame(LLMUpdateSettingsFrame(update=OpenAILLMSettings(temperature=0.1))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zi-update-settings-openai-llm.py b/examples/foundational/55zi-update-settings-openai-llm.py new file mode 100644 index 000000000..a8c253bc2 --- /dev/null +++ b/examples/foundational/55zi-update-settings-openai-llm.py @@ -0,0 +1,126 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.base_llm import OpenAILLMSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating OpenAI LLM settings: temperature=0.1") + await task.queue_frame(LLMUpdateSettingsFrame(update=OpenAILLMSettings(temperature=0.1))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zj-update-settings-anthropic-llm.py b/examples/foundational/55zj-update-settings-anthropic-llm.py new file mode 100644 index 000000000..4c8341a6a --- /dev/null +++ b/examples/foundational/55zj-update-settings-anthropic-llm.py @@ -0,0 +1,125 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.anthropic.llm import AnthropicLLMService, AnthropicLLMSettings +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = AnthropicLLMService(api_key=os.getenv("ANTHROPIC_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Anthropic LLM settings: temperature=0.1") + await task.queue_frame(LLMUpdateSettingsFrame(update=AnthropicLLMSettings(temperature=0.1))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zk-update-settings-google-llm.py b/examples/foundational/55zk-update-settings-google-llm.py new file mode 100644 index 000000000..140c0fccb --- /dev/null +++ b/examples/foundational/55zk-update-settings-google-llm.py @@ -0,0 +1,125 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.google.llm import GoogleLLMService, GoogleLLMSettings +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Google LLM settings: temperature=0.1") + await task.queue_frame(LLMUpdateSettingsFrame(update=GoogleLLMSettings(temperature=0.1))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zk-update-settings-google-vertex-llm.py b/examples/foundational/55zk-update-settings-google-vertex-llm.py new file mode 100644 index 000000000..41c0b8a37 --- /dev/null +++ b/examples/foundational/55zk-update-settings-google-vertex-llm.py @@ -0,0 +1,130 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.google.llm import GoogleLLMSettings +from pipecat.services.google.llm_vertex import GoogleVertexLLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = GoogleVertexLLMService( + credentials=os.getenv("GOOGLE_VERTEX_TEST_CREDENTIALS"), + project_id=os.getenv("GOOGLE_CLOUD_PROJECT_ID"), + location=os.getenv("GOOGLE_CLOUD_LOCATION"), + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Google Vertex LLM settings: temperature=0.1") + await task.queue_frame(LLMUpdateSettingsFrame(update=GoogleLLMSettings(temperature=0.1))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zl-update-settings-azure-realtime.py b/examples/foundational/55zl-update-settings-azure-realtime.py new file mode 100644 index 000000000..b8f049db0 --- /dev/null +++ b/examples/foundational/55zl-update-settings-azure-realtime.py @@ -0,0 +1,140 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + AssistantTurnStoppedMessage, + LLMContextAggregatorPair, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.azure.realtime.llm import AzureRealtimeLLMService +from pipecat.services.openai.realtime import events +from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMSettings +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + llm = AzureRealtimeLLMService( + api_key=os.getenv("AZURE_REALTIME_API_KEY"), + base_url=os.getenv("AZURE_REALTIME_BASE_URL"), + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), + user_aggregator, + llm, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @assistant_aggregator.event_handler("on_assistant_turn_stopped") + async def on_assistant_turn_stopped(aggregator, message: AssistantTurnStoppedMessage): + timestamp = f"[{message.timestamp}] " if message.timestamp else "" + line = f"{timestamp}assistant: {message.content}" + logger.info(f"Transcript: {line}") + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Azure Realtime LLM settings: output_modalities=['text']") + await task.queue_frame( + LLMUpdateSettingsFrame( + update=OpenAIRealtimeLLMSettings( + session_properties=events.SessionProperties(output_modalities=["text"]) + ) + ) + ) + + await asyncio.sleep(10) + logger.info("Updating Azure Realtime LLM settings: output_modalities=['audio']") + await task.queue_frame( + LLMUpdateSettingsFrame( + update=OpenAIRealtimeLLMSettings( + session_properties=events.SessionProperties(output_modalities=["audio"]) + ) + ) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zl-update-settings-openai-realtime.py b/examples/foundational/55zl-update-settings-openai-realtime.py new file mode 100644 index 000000000..9c18d528e --- /dev/null +++ b/examples/foundational/55zl-update-settings-openai-realtime.py @@ -0,0 +1,139 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + AssistantTurnStoppedMessage, + LLMContextAggregatorPair, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.openai.realtime import events +from pipecat.services.openai.realtime.llm import ( + OpenAIRealtimeLLMService, + OpenAIRealtimeLLMSettings, +) +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + llm = OpenAIRealtimeLLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), + user_aggregator, + llm, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @assistant_aggregator.event_handler("on_assistant_turn_stopped") + async def on_assistant_turn_stopped(aggregator, message: AssistantTurnStoppedMessage): + timestamp = f"[{message.timestamp}] " if message.timestamp else "" + line = f"{timestamp}assistant: {message.content}" + logger.info(f"Transcript: {line}") + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating OpenAI Realtime LLM settings: output_modalities=['text']") + await task.queue_frame( + LLMUpdateSettingsFrame( + update=OpenAIRealtimeLLMSettings( + session_properties=events.SessionProperties(output_modalities=["text"]) + ) + ) + ) + + await asyncio.sleep(10) + logger.info("Updating OpenAI Realtime LLM settings: output_modalities=['audio']") + await task.queue_frame( + LLMUpdateSettingsFrame( + update=OpenAIRealtimeLLMSettings( + session_properties=events.SessionProperties(output_modalities=["audio"]) + ) + ) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zm-update-settings-gemini-live-vertex.py b/examples/foundational/55zm-update-settings-gemini-live-vertex.py new file mode 100644 index 000000000..575fbe090 --- /dev/null +++ b/examples/foundational/55zm-update-settings-gemini-live-vertex.py @@ -0,0 +1,117 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.google.gemini_live.llm import GeminiLiveLLMSettings +from pipecat.services.google.gemini_live.llm_vertex import GeminiLiveVertexLLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + llm = GeminiLiveVertexLLMService( + credentials=os.getenv("GOOGLE_VERTEX_TEST_CREDENTIALS"), + project_id=os.getenv("GOOGLE_CLOUD_PROJECT_ID"), + location=os.getenv("GOOGLE_CLOUD_LOCATION"), + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), + user_aggregator, + llm, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Gemini Live Vertex LLM settings: temperature=0.1") + await task.queue_frame( + LLMUpdateSettingsFrame(update=GeminiLiveLLMSettings(temperature=0.1)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zm-update-settings-gemini-live.py b/examples/foundational/55zm-update-settings-gemini-live.py new file mode 100644 index 000000000..8ad635fd5 --- /dev/null +++ b/examples/foundational/55zm-update-settings-gemini-live.py @@ -0,0 +1,115 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.google.gemini_live.llm import ( + GeminiLiveLLMService, + GeminiLiveLLMSettings, +) +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + llm = GeminiLiveLLMService(api_key=os.getenv("GOOGLE_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), + user_aggregator, + llm, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Gemini Live LLM settings: temperature=0.1") + await task.queue_frame( + LLMUpdateSettingsFrame(update=GeminiLiveLLMSettings(temperature=0.1)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zn-update-settings-ultravox-realtime.py b/examples/foundational/55zn-update-settings-ultravox-realtime.py new file mode 100644 index 000000000..967d40741 --- /dev/null +++ b/examples/foundational/55zn-update-settings-ultravox-realtime.py @@ -0,0 +1,143 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import datetime +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + AssistantTurnStoppedMessage, + LLMContextAggregatorPair, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.ultravox.llm import ( + OneShotInputParams, + UltravoxRealtimeLLMService, + UltravoxRealtimeLLMSettings, +) +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + system_prompt = "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way." + + llm = UltravoxRealtimeLLMService( + params=OneShotInputParams( + api_key=os.getenv("ULTRAVOX_API_KEY"), + system_prompt=system_prompt, + temperature=0.3, + max_duration=datetime.timedelta(minutes=3), + ), + one_shot_selected_tools=ToolsSchema(standard_tools=[]), + ) + + messages = [ + { + "role": "system", + "content": system_prompt, + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), + user_aggregator, + llm, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @assistant_aggregator.event_handler("on_assistant_turn_stopped") + async def on_assistant_turn_stopped(aggregator, message: AssistantTurnStoppedMessage): + timestamp = f"[{message.timestamp}] " if message.timestamp else "" + line = f"{timestamp}assistant: {message.content}" + logger.info(f"Transcript: {line}") + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Ultravox Realtime LLM settings: output_medium=text") + await task.queue_frame( + LLMUpdateSettingsFrame(update=UltravoxRealtimeLLMSettings(output_medium="text")) + ) + + await asyncio.sleep(10) + logger.info("Updating Ultravox Realtime LLM settings: output_medium=voice") + await task.queue_frame( + LLMUpdateSettingsFrame(update=UltravoxRealtimeLLMSettings(output_medium="voice")) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zo-update-settings-grok-realtime.py b/examples/foundational/55zo-update-settings-grok-realtime.py new file mode 100644 index 000000000..7d7370f7b --- /dev/null +++ b/examples/foundational/55zo-update-settings-grok-realtime.py @@ -0,0 +1,129 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + AssistantTurnStoppedMessage, + LLMContextAggregatorPair, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.grok.realtime import events +from pipecat.services.grok.realtime.llm import ( + GrokRealtimeLLMService, + GrokRealtimeLLMSettings, +) +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + llm = GrokRealtimeLLMService(api_key=os.getenv("GROK_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), + user_aggregator, + llm, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @assistant_aggregator.event_handler("on_assistant_turn_stopped") + async def on_assistant_turn_stopped(aggregator, message: AssistantTurnStoppedMessage): + timestamp = f"[{message.timestamp}] " if message.timestamp else "" + line = f"{timestamp}assistant: {message.content}" + logger.info(f"Transcript: {line}") + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Grok Realtime LLM settings: voice='Rex'") + await task.queue_frame( + LLMUpdateSettingsFrame( + update=GrokRealtimeLLMSettings( + session_properties=events.SessionProperties(voice="Rex") + ) + ) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zp-update-settings-aws-bedrock-llm.py b/examples/foundational/55zp-update-settings-aws-bedrock-llm.py new file mode 100644 index 000000000..1c2781e72 --- /dev/null +++ b/examples/foundational/55zp-update-settings-aws-bedrock-llm.py @@ -0,0 +1,131 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.aws.llm import AWSBedrockLLMService, AWSBedrockLLMSettings +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = AWSBedrockLLMService( + aws_region="us-west-2", + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", + params=AWSBedrockLLMService.InputParams(temperature=0.8), + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating AWS Bedrock LLM settings: temperature=0.1") + await task.queue_frame( + LLMUpdateSettingsFrame(update=AWSBedrockLLMSettings(temperature=0.1)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zq-update-settings-fal-stt.py b/examples/foundational/55zq-update-settings-fal-stt.py new file mode 100644 index 000000000..9792961f2 --- /dev/null +++ b/examples/foundational/55zq-update-settings-fal-stt.py @@ -0,0 +1,125 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, STTUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.fal.stt import FalSTTService, FalSTTSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = FalSTTService(api_key=os.getenv("FAL_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info('Updating Fal STT settings: task="translate"') + await task.queue_frame(STTUpdateSettingsFrame(update=FalSTTSettings(task="translate"))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zr-update-settings-gradium-stt.py b/examples/foundational/55zr-update-settings-gradium-stt.py new file mode 100644 index 000000000..6a1a25c3c --- /dev/null +++ b/examples/foundational/55zr-update-settings-gradium-stt.py @@ -0,0 +1,128 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, STTUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.gradium.stt import GradiumSTTService, GradiumSTTSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = GradiumSTTService( + api_key=os.getenv("GRADIUM_API_KEY"), + api_endpoint_base_url="wss://us.api.gradium.ai/api/speech/asr", + ) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Gradium STT settings: delay_in_frames=5") + await task.queue_frame(STTUpdateSettingsFrame(update=GradiumSTTSettings(delay_in_frames=5))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zs-update-settings-hathora-stt.py b/examples/foundational/55zs-update-settings-hathora-stt.py new file mode 100644 index 000000000..f3aca9c89 --- /dev/null +++ b/examples/foundational/55zs-update-settings-hathora-stt.py @@ -0,0 +1,129 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, STTUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.hathora.stt import HathoraSTTService, HathoraSTTSettings +from pipecat.services.hathora.utils import ConfigOption +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = HathoraSTTService(api_key=os.getenv("HATHORA_API_KEY"), model="deepgram-nova3") + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Hathora STT settings: language=es") + await task.queue_frame( + STTUpdateSettingsFrame(update=HathoraSTTSettings(language=Language.ES)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zt-update-settings-nvidia-segmented-stt.py b/examples/foundational/55zt-update-settings-nvidia-segmented-stt.py new file mode 100644 index 000000000..624da149e --- /dev/null +++ b/examples/foundational/55zt-update-settings-nvidia-segmented-stt.py @@ -0,0 +1,127 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, STTUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.nvidia.stt import NvidiaSegmentedSTTService, NvidiaSegmentedSTTSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = NvidiaSegmentedSTTService(api_key=os.getenv("NVIDIA_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating NVIDIA Segmented STT settings: profanity_filter=True") + await task.queue_frame( + STTUpdateSettingsFrame(update=NvidiaSegmentedSTTSettings(profanity_filter=True)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zt-update-settings-nvidia-stt.py b/examples/foundational/55zt-update-settings-nvidia-stt.py new file mode 100644 index 000000000..0e7b6a74a --- /dev/null +++ b/examples/foundational/55zt-update-settings-nvidia-stt.py @@ -0,0 +1,128 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, STTUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.nvidia.stt import NvidiaSTTService, NvidiaSTTSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = NvidiaSTTService(api_key=os.getenv("NVIDIA_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating NVIDIA STT settings: language=es") + await task.queue_frame( + STTUpdateSettingsFrame(update=NvidiaSTTSettings(language=Language.ES)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zu-update-settings-openai-realtime-stt.py b/examples/foundational/55zu-update-settings-openai-realtime-stt.py new file mode 100644 index 000000000..1f1592df7 --- /dev/null +++ b/examples/foundational/55zu-update-settings-openai-realtime-stt.py @@ -0,0 +1,128 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, STTUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.services.openai.stt import OpenAIRealtimeSTTService, OpenAIRealtimeSTTSettings +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = OpenAIRealtimeSTTService(api_key=os.getenv("OPENAI_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating OpenAI Realtime STT settings: language=es") + await task.queue_frame( + STTUpdateSettingsFrame(update=OpenAIRealtimeSTTSettings(language=Language.ES)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zv-update-settings-asyncai-http-tts.py b/examples/foundational/55zv-update-settings-asyncai-http-tts.py new file mode 100644 index 000000000..206a80eed --- /dev/null +++ b/examples/foundational/55zv-update-settings-asyncai-http-tts.py @@ -0,0 +1,133 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +import asyncio +import os + +import aiohttp +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.asyncai.tts import AsyncAIHttpTTSService, AsyncAITTSSettings +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + async with aiohttp.ClientSession() as session: + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = AsyncAIHttpTTSService( + api_key=os.getenv("ASYNCAI_API_KEY", ""), + voice_id=os.getenv("ASYNCAI_VOICE_ID", "e0f39dc4-f691-4e78-bba5-5c636692cc04"), + aiohttp_session=session, + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating AsyncAI HTTP TTS settings: language=es") + await task.queue_frame( + TTSUpdateSettingsFrame(update=AsyncAITTSSettings(language=Language.ES)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zv-update-settings-asyncai-tts.py b/examples/foundational/55zv-update-settings-asyncai-tts.py new file mode 100644 index 000000000..f910e5fe3 --- /dev/null +++ b/examples/foundational/55zv-update-settings-asyncai-tts.py @@ -0,0 +1,128 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.asyncai.tts import AsyncAITTSService, AsyncAITTSSettings +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = AsyncAITTSService( + api_key=os.getenv("ASYNCAI_API_KEY", ""), + voice_id=os.getenv("ASYNCAI_VOICE_ID", "e0f39dc4-f691-4e78-bba5-5c636692cc04"), + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating AsyncAI TTS settings: language=es") + await task.queue_frame( + TTSUpdateSettingsFrame(update=AsyncAITTSSettings(language=Language.ES)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zw-update-settings-gradium-tts.py b/examples/foundational/55zw-update-settings-gradium-tts.py new file mode 100644 index 000000000..39090d5fa --- /dev/null +++ b/examples/foundational/55zw-update-settings-gradium-tts.py @@ -0,0 +1,128 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.gradium.tts import GradiumTTSService, GradiumTTSSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = GradiumTTSService( + api_key=os.getenv("GRADIUM_API_KEY"), + voice_id="YTpq7expH9539ERJ", + url="wss://us.api.gradium.ai/api/speech/tts", + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info('Updating Gradium TTS settings: voice="LFZvm12tW_z0xfGo"') + await task.queue_frame( + TTSUpdateSettingsFrame(update=GradiumTTSSettings(voice="LFZvm12tW_z0xfGo")) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zx-update-settings-cerebras-llm.py b/examples/foundational/55zx-update-settings-cerebras-llm.py new file mode 100644 index 000000000..72aa8518d --- /dev/null +++ b/examples/foundational/55zx-update-settings-cerebras-llm.py @@ -0,0 +1,126 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.cerebras.llm import CerebrasLLMService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.base_llm import OpenAILLMSettings +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = CerebrasLLMService(api_key=os.getenv("CEREBRAS_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Cerebras LLM settings: temperature=0.1") + await task.queue_frame(LLMUpdateSettingsFrame(update=OpenAILLMSettings(temperature=0.1))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zy-update-settings-deepseek-llm.py b/examples/foundational/55zy-update-settings-deepseek-llm.py new file mode 100644 index 000000000..de4e4149e --- /dev/null +++ b/examples/foundational/55zy-update-settings-deepseek-llm.py @@ -0,0 +1,126 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.deepseek.llm import DeepSeekLLMService +from pipecat.services.openai.base_llm import OpenAILLMSettings +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = DeepSeekLLMService(api_key=os.getenv("DEEPSEEK_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating DeepSeek LLM settings: temperature=0.1") + await task.queue_frame(LLMUpdateSettingsFrame(update=OpenAILLMSettings(temperature=0.1))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zz-update-settings-fireworks-llm.py b/examples/foundational/55zz-update-settings-fireworks-llm.py new file mode 100644 index 000000000..d864cacb2 --- /dev/null +++ b/examples/foundational/55zz-update-settings-fireworks-llm.py @@ -0,0 +1,129 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.fireworks.llm import FireworksLLMService +from pipecat.services.openai.base_llm import OpenAILLMSettings +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = FireworksLLMService( + api_key=os.getenv("FIREWORKS_API_KEY"), + model="accounts/fireworks/models/gpt-oss-20b", + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Fireworks LLM settings: temperature=0.1") + await task.queue_frame(LLMUpdateSettingsFrame(update=OpenAILLMSettings(temperature=0.1))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zza-update-settings-grok-llm.py b/examples/foundational/55zza-update-settings-grok-llm.py new file mode 100644 index 000000000..dbf07f21d --- /dev/null +++ b/examples/foundational/55zza-update-settings-grok-llm.py @@ -0,0 +1,126 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.grok.llm import GrokLLMService +from pipecat.services.openai.base_llm import OpenAILLMSettings +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = GrokLLMService(api_key=os.getenv("GROK_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Grok LLM settings: temperature=0.1") + await task.queue_frame(LLMUpdateSettingsFrame(update=OpenAILLMSettings(temperature=0.1))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zzb-update-settings-groq-llm.py b/examples/foundational/55zzb-update-settings-groq-llm.py new file mode 100644 index 000000000..8244f611a --- /dev/null +++ b/examples/foundational/55zzb-update-settings-groq-llm.py @@ -0,0 +1,128 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.groq.llm import GroqLLMService +from pipecat.services.openai.base_llm import OpenAILLMSettings +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = GroqLLMService( + api_key=os.getenv("GROQ_API_KEY"), model="meta-llama/llama-4-maverick-17b-128e-instruct" + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Groq LLM settings: temperature=0.1") + await task.queue_frame(LLMUpdateSettingsFrame(update=OpenAILLMSettings(temperature=0.1))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zzc-update-settings-mistral-llm.py b/examples/foundational/55zzc-update-settings-mistral-llm.py new file mode 100644 index 000000000..642eda3c5 --- /dev/null +++ b/examples/foundational/55zzc-update-settings-mistral-llm.py @@ -0,0 +1,126 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.mistral.llm import MistralLLMService +from pipecat.services.openai.base_llm import OpenAILLMSettings +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = MistralLLMService(api_key=os.getenv("MISTRAL_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Mistral LLM settings: temperature=0.1") + await task.queue_frame(LLMUpdateSettingsFrame(update=OpenAILLMSettings(temperature=0.1))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zzd-update-settings-nvidia-llm.py b/examples/foundational/55zzd-update-settings-nvidia-llm.py new file mode 100644 index 000000000..5ffa0ff23 --- /dev/null +++ b/examples/foundational/55zzd-update-settings-nvidia-llm.py @@ -0,0 +1,128 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.nvidia.llm import NvidiaLLMService +from pipecat.services.openai.base_llm import OpenAILLMSettings +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = NvidiaLLMService( + api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.1-405b-instruct" + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating NVIDIA LLM settings: temperature=0.1") + await task.queue_frame(LLMUpdateSettingsFrame(update=OpenAILLMSettings(temperature=0.1))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zze-update-settings-ollama-llm.py b/examples/foundational/55zze-update-settings-ollama-llm.py new file mode 100644 index 000000000..ca3714943 --- /dev/null +++ b/examples/foundational/55zze-update-settings-ollama-llm.py @@ -0,0 +1,126 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.ollama.llm import OLLamaLLMService +from pipecat.services.openai.base_llm import OpenAILLMSettings +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OLLamaLLMService(model="llama3.2") # Update to the model you're running locally + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating OLLama LLM settings: temperature=0.1") + await task.queue_frame(LLMUpdateSettingsFrame(update=OpenAILLMSettings(temperature=0.1))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zzf-update-settings-openrouter-llm.py b/examples/foundational/55zzf-update-settings-openrouter-llm.py new file mode 100644 index 000000000..90606a572 --- /dev/null +++ b/examples/foundational/55zzf-update-settings-openrouter-llm.py @@ -0,0 +1,126 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.base_llm import OpenAILLMSettings +from pipecat.services.openrouter.llm import OpenRouterLLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenRouterLLMService(api_key=os.getenv("OPENROUTER_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating OpenRouter LLM settings: temperature=0.1") + await task.queue_frame(LLMUpdateSettingsFrame(update=OpenAILLMSettings(temperature=0.1))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zzg-update-settings-perplexity-llm.py b/examples/foundational/55zzg-update-settings-perplexity-llm.py new file mode 100644 index 000000000..771b1c794 --- /dev/null +++ b/examples/foundational/55zzg-update-settings-perplexity-llm.py @@ -0,0 +1,125 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.base_llm import OpenAILLMSettings +from pipecat.services.perplexity.llm import PerplexityLLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = PerplexityLLMService(api_key=os.getenv("PERPLEXITY_API_KEY")) + + messages = [ + { + "role": "user", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way. Start by introducing yourself.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Perplexity LLM settings: temperature=0.1") + await task.queue_frame(LLMUpdateSettingsFrame(update=OpenAILLMSettings(temperature=0.1))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zzh-update-settings-qwen-llm.py b/examples/foundational/55zzh-update-settings-qwen-llm.py new file mode 100644 index 000000000..81ace2117 --- /dev/null +++ b/examples/foundational/55zzh-update-settings-qwen-llm.py @@ -0,0 +1,126 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.base_llm import OpenAILLMSettings +from pipecat.services.qwen.llm import QwenLLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = QwenLLMService(api_key=os.getenv("QWEN_API_KEY"), model="qwen2.5-72b-instruct") + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Qwen LLM settings: temperature=0.1") + await task.queue_frame(LLMUpdateSettingsFrame(update=OpenAILLMSettings(temperature=0.1))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zzi-update-settings-sambanova-llm.py b/examples/foundational/55zzi-update-settings-sambanova-llm.py new file mode 100644 index 000000000..82382a6bd --- /dev/null +++ b/examples/foundational/55zzi-update-settings-sambanova-llm.py @@ -0,0 +1,126 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.base_llm import OpenAILLMSettings +from pipecat.services.sambanova.llm import SambaNovaLLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = SambaNovaLLMService(api_key=os.getenv("SAMBANOVA_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating SambaNova LLM settings: temperature=0.1") + await task.queue_frame(LLMUpdateSettingsFrame(update=OpenAILLMSettings(temperature=0.1))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zzj-update-settings-together-llm.py b/examples/foundational/55zzj-update-settings-together-llm.py new file mode 100644 index 000000000..1f0a0557f --- /dev/null +++ b/examples/foundational/55zzj-update-settings-together-llm.py @@ -0,0 +1,129 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.base_llm import OpenAILLMSettings +from pipecat.services.together.llm import TogetherLLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = TogetherLLMService( + api_key=os.getenv("TOGETHER_API_KEY"), + model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo", + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating Together LLM settings: temperature=0.1") + await task.queue_frame(LLMUpdateSettingsFrame(update=OpenAILLMSettings(temperature=0.1))) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zzk-update-settings-aws-nova-sonic-llm.py b/examples/foundational/55zzk-update-settings-aws-nova-sonic-llm.py new file mode 100644 index 000000000..1faafdbac --- /dev/null +++ b/examples/foundational/55zzk-update-settings-aws-nova-sonic-llm.py @@ -0,0 +1,124 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.aws.nova_sonic.llm import AWSNovaSonicLLMService, AWSNovaSonicLLMSettings +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + llm = AWSNovaSonicLLMService( + secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), + access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), + region=os.getenv("AWS_REGION"), + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + { + "role": "user", + "content": "Tell me a fun fact!", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + user_aggregator, + llm, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info("Updating AWS Nova Sonic LLM settings: temperature=0.1") + await task.queue_frame( + LLMUpdateSettingsFrame(update=AWSNovaSonicLLMSettings(temperature=0.1)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zzl-update-settings-nvidia-tts.py b/examples/foundational/55zzl-update-settings-nvidia-tts.py new file mode 100644 index 000000000..b92651496 --- /dev/null +++ b/examples/foundational/55zzl-update-settings-nvidia-tts.py @@ -0,0 +1,125 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.nvidia.tts import NvidiaTTSService, NvidiaTTSSettings +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = NvidiaTTSService(api_key=os.getenv("NVIDIA_API_KEY")) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info('Updating NVIDIA TTS settings: language="ES_US"') + await task.queue_frame( + TTSUpdateSettingsFrame(update=NvidiaTTSSettings(language=Language.ES_US)) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/55zzm-update-settings-speechmatics-tts.py b/examples/foundational/55zzm-update-settings-speechmatics-tts.py new file mode 100644 index 000000000..36b66fe53 --- /dev/null +++ b/examples/foundational/55zzm-update-settings-speechmatics-tts.py @@ -0,0 +1,129 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +import aiohttp +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.services.speechmatics.tts import SpeechmaticsTTSService, SpeechmaticsTTSSettings +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + async with aiohttp.ClientSession() as session: + tts = SpeechmaticsTTSService( + api_key=os.getenv("SPEECHMATICS_API_KEY"), + aiohttp_session=session, + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + await asyncio.sleep(10) + logger.info('Updating Speechmatics TTS settings: voice="theo"') + await task.queue_frame( + TTSUpdateSettingsFrame(update=SpeechmaticsTTSSettings(voice="theo")) + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 2d4bbc65e..79aedb771 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -42,6 +42,7 @@ from pipecat.utils.utils import obj_count, obj_id if TYPE_CHECKING: from pipecat.processors.aggregators.llm_context import LLMContext, NotGiven from pipecat.processors.frame_processor import FrameProcessor + from pipecat.services.settings import ServiceSettings from pipecat.utils.tracing.tracing_context import TracingContext @@ -2120,13 +2121,21 @@ class TTSStoppedFrame(ControlFrame): class ServiceUpdateSettingsFrame(ControlFrame): """Base frame for updating service settings. - A control frame containing a request to update service settings. + Supports both a ``settings`` dict (for backward compatibility) and an + ``update`` object. When both are provided, ``update`` takes precedence. Parameters: settings: Dictionary of setting name to value mappings. + + .. deprecated:: 0.0.103 + Use ``update`` with a typed settings object instead. + + update: :class:`~pipecat.services.settings.ServiceSettings` object + describing the delta to apply. """ - settings: Mapping[str, Any] + settings: Mapping[str, Any] = field(default_factory=dict) + update: Optional["ServiceSettings"] = None @dataclass diff --git a/src/pipecat/services/ai_service.py b/src/pipecat/services/ai_service.py index 52b42663f..8b32d7222 100644 --- a/src/pipecat/services/ai_service.py +++ b/src/pipecat/services/ai_service.py @@ -10,7 +10,7 @@ Provides the foundation for all AI services in the Pipecat framework, including model management, settings handling, and frame processing lifecycle methods. """ -from typing import Any, AsyncGenerator, Dict, Mapping +from typing import Any, AsyncGenerator, Dict from loguru import logger @@ -23,6 +23,7 @@ from pipecat.frames.frames import ( ) from pipecat.metrics.metrics import MetricsData from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.services.settings import ServiceSettings class AIService(FrameProcessor): @@ -41,29 +42,27 @@ class AIService(FrameProcessor): **kwargs: Additional arguments passed to the parent FrameProcessor. """ super().__init__(**kwargs) - self._model_name: str = "" - self._settings: Dict[str, Any] = {} + self._settings: ServiceSettings = ServiceSettings(model="") self._session_properties: Dict[str, Any] = {} self._tracing_enabled: bool = False self._tracing_context = None - @property - def model_name(self) -> str: - """Get the current model name. + def _sync_model_name_to_metrics(self): + """Sync the current AI model name (in `self._settings.model`) for usage in metrics. - Returns: - The name of the AI model being used. - """ - return self._model_name + We don't store model name here because there's already a single source + of truth for it in `self._settings.model`. This method is just for + syncing the model name to the metrics data. - def set_model_name(self, model: str): - """Set the AI model name and update metrics. + TODO: as a next step we should make it so that service classes pass + model into `super().__init__` and `AIService` can be responsible for + syncing its initial value to metrics, just as it's responsible for + syncing any updates to its value to metrics via `_update_settings`. Args: model: The name of the AI model to use. """ - self._model_name = model - self.set_core_metrics_data(MetricsData(processor=self.name, model=self._model_name)) + self.set_core_metrics_data(MetricsData(processor=self.name, model=self._settings.model)) async def start(self, frame: StartFrame): """Start the AI service. @@ -99,44 +98,45 @@ class AIService(FrameProcessor): """ pass - async def _update_settings(self, settings: Mapping[str, Any]): - from pipecat.services.openai.realtime.events import SessionProperties + async def _update_settings(self, update: ServiceSettings) -> Dict[str, Any]: + """Apply a settings update and return the changed fields. - for key, value in settings.items(): - logger.debug("Update request for:", key, value) + The update is applied to ``_settings`` and a dict mapping each changed + field name to its **pre-update** value is returned. The ``model`` + field is handled specially: when it changes, ``set_model_name`` is + called. - if key in self._settings: - logger.info(f"Updating LLM setting {key} to: [{value}]") - self._settings[key] = value - elif key in SessionProperties.model_fields: - logger.debug("Attempting to update", key, value) + Concrete services should override this method (calling ``super()``) + to react to specific changed fields (e.g. reconnect on voice change). - try: - from pipecat.services.openai.realtime.events import TurnDetection + Args: + update: A settings delta. - if isinstance(self._session_properties, SessionProperties): - current_properties = self._session_properties - else: - current_properties = SessionProperties(**self._session_properties) + Returns: + Dict mapping changed field names to their previous values. + """ + changed = self._settings.apply_update(update) - if key == "turn_detection" and isinstance(value, dict): - turn_detection = TurnDetection(**value) - setattr(current_properties, key, turn_detection) - else: - setattr(current_properties, key, value) + if "model" in changed: + self._sync_model_name_to_metrics() - validated_properties = SessionProperties.model_validate( - current_properties.model_dump() - ) - logger.info(f"Updating LLM setting {key} to: [{value}]") - self._session_properties = validated_properties.model_dump() - except Exception as e: - logger.warning(f"Unexpected error updating session property {key}: {e}") - elif key == "model": - logger.info(f"Updating LLM setting {key} to: [{value}]") - self.set_model_name(value) - else: - logger.warning(f"Unknown setting for {self.name} service: {key}") + if changed: + logger.info(f"{self.name}: updated settings fields: {set(changed)}") + + return changed + + def _warn_unhandled_updated_settings(self, unhandled): + """Log a warning for settings changes that won't take effect at runtime. + + Convenience helper for ``_update_settings`` overrides. Accepts any + iterable of field names (a ``dict``, ``set``, ``dict_keys``, etc.). + + Args: + unhandled: Field names that changed but are not applied. + """ + if unhandled: + fields = ", ".join(sorted(unhandled)) + logger.warning(f"{self.name}: runtime update of [{fields}] is not currently supported") async def process_frame(self, frame: Frame, direction: FrameDirection): """Process frames and handle service lifecycle. diff --git a/src/pipecat/services/anthropic/llm.py b/src/pipecat/services/anthropic/llm.py index e715c242d..e79792bb0 100644 --- a/src/pipecat/services/anthropic/llm.py +++ b/src/pipecat/services/anthropic/llm.py @@ -16,8 +16,8 @@ import copy import io import json import re -from dataclasses import dataclass -from typing import Any, Dict, List, Literal, Optional, Union +from dataclasses import dataclass, field +from typing import Any, ClassVar, Dict, List, Literal, Optional, Union import httpx from loguru import logger @@ -42,7 +42,6 @@ from pipecat.frames.frames import ( LLMThoughtEndFrame, LLMThoughtStartFrame, LLMThoughtTextFrame, - LLMUpdateSettingsFrame, UserImageRawFrame, ) from pipecat.metrics.metrics import LLMTokenUsage @@ -59,6 +58,8 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import FunctionCallFromLLM, LLMService +from pipecat.services.settings import NOT_GIVEN as _NOT_GIVEN +from pipecat.services.settings import LLMSettings, _NotGiven, is_given from pipecat.utils.tracing.service_decorators import traced_llm try: @@ -69,6 +70,50 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +class AnthropicThinkingConfig(BaseModel): + """Configuration for extended thinking. + + Parameters: + type: Type of thinking mode (currently only "enabled" or "disabled"). + budget_tokens: Maximum number of tokens for thinking. + With today's models, the minimum is 1024. + Only allowed if type is "enabled". + """ + + # Why `| str` here? To not break compatibility in case Anthropic adds + # more types in the future. + type: Literal["enabled", "disabled"] | str + + # Why not enforce minimnum of 1024 here? To not break compatibility in + # case Anthropic changes this requirement in the future. + budget_tokens: int + + +@dataclass +class AnthropicLLMSettings(LLMSettings): + """Settings for Anthropic LLM services. + + Parameters: + enable_prompt_caching: Whether to enable prompt caching. + thinking: Extended thinking configuration. + """ + + enable_prompt_caching: bool | _NotGiven = field(default_factory=lambda: _NOT_GIVEN) + thinking: AnthropicThinkingConfig | _NotGiven = field(default_factory=lambda: _NOT_GIVEN) + + @classmethod + def from_mapping(cls, settings): + """Convert a plain dict to settings, coercing thinking dicts. + + For backward compatibility, a ``thinking`` value that is a plain dict + is converted to a :class:`AnthropicThinkingConfig`. + """ + instance = super().from_mapping(settings) + if is_given(instance.thinking) and isinstance(instance.thinking, dict): + instance.thinking = AnthropicThinkingConfig(**instance.thinking) + return instance + + @dataclass class AnthropicContextAggregatorPair: """Pair of context aggregators for Anthropic conversations. @@ -115,26 +160,13 @@ class AnthropicLLMService(LLMService): Can use custom clients like AsyncAnthropicBedrock and AsyncAnthropicVertex. """ + _settings: AnthropicLLMSettings + # Overriding the default adapter to use the Anthropic one. adapter_class = AnthropicLLMAdapter - class ThinkingConfig(BaseModel): - """Configuration for extended thinking. - - Parameters: - type: Type of thinking mode (currently only "enabled" or "disabled"). - budget_tokens: Maximum number of tokens for thinking. - With today's models, the minimum is 1024. - Only allowed if type is "enabled". - """ - - # Why `| str` here? To not break compatibility in case Anthropic adds - # more types in the future. - type: Literal["enabled", "disabled"] | str - - # Why not enforce minimnum of 1024 here? To not break compatibility in - # case Anthropic changes this requirement in the future. - budget_tokens: int + # Backward compatibility: ThinkingConfig used to be defined inline here. + ThinkingConfig = AnthropicThinkingConfig class InputParams(BaseModel): """Input parameters for Anthropic model inference. @@ -163,9 +195,7 @@ class AnthropicLLMService(LLMService): temperature: Optional[float] = Field(default_factory=lambda: NOT_GIVEN, ge=0.0, le=1.0) top_k: Optional[int] = Field(default_factory=lambda: NOT_GIVEN, ge=0) top_p: Optional[float] = Field(default_factory=lambda: NOT_GIVEN, ge=0.0, le=1.0) - thinking: Optional["AnthropicLLMService.ThinkingConfig"] = Field( - default_factory=lambda: NOT_GIVEN - ) + thinking: Optional[AnthropicThinkingConfig] = Field(default_factory=lambda: NOT_GIVEN) extra: Optional[Dict[str, Any]] = Field(default_factory=dict) def model_post_init(self, __context): @@ -207,12 +237,12 @@ class AnthropicLLMService(LLMService): self._client = client or AsyncAnthropic( api_key=api_key ) # if the client is provided, use it and remove it, otherwise create a new one - self.set_model_name(model) self._retry_timeout_secs = retry_timeout_secs self._retry_on_timeout = retry_on_timeout - self._settings = { - "max_tokens": params.max_tokens, - "enable_prompt_caching": ( + self._settings = AnthropicLLMSettings( + model=model, + max_tokens=params.max_tokens, + enable_prompt_caching=( params.enable_prompt_caching if params.enable_prompt_caching is not None else ( @@ -221,12 +251,13 @@ class AnthropicLLMService(LLMService): else False ) ), - "temperature": params.temperature, - "top_k": params.top_k, - "top_p": params.top_p, - "thinking": params.thinking, - "extra": params.extra if isinstance(params.extra, dict) else {}, - } + temperature=params.temperature, + top_k=params.top_k, + top_p=params.top_p, + thinking=params.thinking, + extra=params.extra if isinstance(params.extra, dict) else {}, + ) + self._sync_model_name_to_metrics() def can_generate_metrics(self) -> bool: """Check if this service can generate usage metrics. @@ -280,7 +311,7 @@ class AnthropicLLMService(LLMService): if isinstance(context, LLMContext): adapter: AnthropicLLMAdapter = self.get_llm_adapter() invocation_params = adapter.get_llm_invocation_params( - context, enable_prompt_caching=self._settings["enable_prompt_caching"] + context, enable_prompt_caching=self._settings.enable_prompt_caching ) messages = invocation_params["messages"] system = invocation_params["system"] @@ -293,21 +324,21 @@ class AnthropicLLMService(LLMService): # Build params using the same method as streaming completions params = { - "model": self.model_name, - "max_tokens": max_tokens if max_tokens is not None else self._settings["max_tokens"], + "model": self._settings.model, + "max_tokens": max_tokens if max_tokens is not None else self._settings.max_tokens, "stream": False, - "temperature": self._settings["temperature"], - "top_k": self._settings["top_k"], - "top_p": self._settings["top_p"], + "temperature": self._settings.temperature, + "top_k": self._settings.top_k, + "top_p": self._settings.top_p, "messages": messages, "system": system, "tools": tools, "betas": ["interleaved-thinking-2025-05-14"], } - if self._settings["thinking"]: - params["thinking"] = self._settings["thinking"].model_dump(exclude_unset=True) + if self._settings.thinking: + params["thinking"] = self._settings.thinking.model_dump(exclude_unset=True) - params.update(self._settings["extra"]) + params.update(self._settings.extra) # LLM completion response = await self._client.beta.messages.create(**params) @@ -358,14 +389,14 @@ class AnthropicLLMService(LLMService): if isinstance(context, LLMContext): adapter: AnthropicLLMAdapter = self.get_llm_adapter() params = adapter.get_llm_invocation_params( - context, enable_prompt_caching=self._settings["enable_prompt_caching"] + context, enable_prompt_caching=self._settings.enable_prompt_caching ) return params # Anthropic-specific context messages = ( context.get_messages_with_cache_control_markers() - if self._settings["enable_prompt_caching"] + if self._settings.enable_prompt_caching else context.messages ) return AnthropicLLMInvocationParams( @@ -407,22 +438,22 @@ class AnthropicLLMService(LLMService): await self.start_ttfb_metrics() params = { - "model": self.model_name, - "max_tokens": self._settings["max_tokens"], + "model": self._settings.model, + "max_tokens": self._settings.max_tokens, "stream": True, - "temperature": self._settings["temperature"], - "top_k": self._settings["top_k"], - "top_p": self._settings["top_p"], + "temperature": self._settings.temperature, + "top_k": self._settings.top_k, + "top_p": self._settings.top_p, } # Add thinking parameter if set - if self._settings["thinking"]: - params["thinking"] = self._settings["thinking"].model_dump(exclude_unset=True) + if self._settings.thinking: + params["thinking"] = self._settings.thinking.model_dump(exclude_unset=True) # Messages, system, tools params.update(params_from_context) - params.update(self._settings["extra"]) + params.update(self._settings.extra) # "Interleaved thinking" needed to allow thinking between sequences # of function calls, when extended thinking is enabled. @@ -576,11 +607,9 @@ class AnthropicLLMService(LLMService): # NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal # LLMContext with it context = AnthropicLLMContext.from_messages(frame.messages) - elif isinstance(frame, LLMUpdateSettingsFrame): - await self._update_settings(frame.settings) elif isinstance(frame, LLMEnablePromptCachingFrame): logger.debug(f"Setting enable prompt caching to: [{frame.enable}]") - self._settings["enable_prompt_caching"] = frame.enable + self._settings.enable_prompt_caching = frame.enable else: await self.push_frame(frame, direction) diff --git a/src/pipecat/services/assemblyai/stt.py b/src/pipecat/services/assemblyai/stt.py index 41a0ae2a0..6a33b6a20 100644 --- a/src/pipecat/services/assemblyai/stt.py +++ b/src/pipecat/services/assemblyai/stt.py @@ -12,6 +12,7 @@ WebSocket API for streaming audio transcription. import asyncio import json +from dataclasses import dataclass, field from typing import Any, AsyncGenerator, Dict, Optional from urllib.parse import urlencode @@ -29,6 +30,7 @@ from pipecat.frames.frames import ( VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven from pipecat.services.stt_latency import ASSEMBLYAI_TTFS_P99 from pipecat.services.stt_service import WebsocketSTTService from pipecat.transcriptions.language import Language @@ -52,6 +54,21 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class AssemblyAISTTSettings(STTSettings): + """Settings for the AssemblyAI STT service. + + See :class:`AssemblyAIConnectionParams` for detailed parameter descriptions. + + Parameters: + connection_params: Connection configuration parameters. + """ + + connection_params: AssemblyAIConnectionParams | _NotGiven = field( + default_factory=lambda: NOT_GIVEN + ) + + class AssemblyAISTTService(WebsocketSTTService): """AssemblyAI real-time speech-to-text service. @@ -60,6 +77,8 @@ class AssemblyAISTTService(WebsocketSTTService): for audio processing and connection management. """ + _settings: AssemblyAISTTSettings + def __init__( self, *, @@ -96,9 +115,11 @@ class AssemblyAISTTService(WebsocketSTTService): ) self._api_key = api_key - self._language = language + self._settings = AssemblyAISTTSettings( + language=language, + connection_params=connection_params, + ) self._api_endpoint_base_url = api_endpoint_base_url - self._connection_params = connection_params self._vad_force_turn_endpoint = vad_force_turn_endpoint self._termination_event = asyncio.Event() @@ -165,6 +186,37 @@ class AssemblyAISTTService(WebsocketSTTService): """ return True + async def _update_settings(self, update: STTSettings) -> dict[str, Any]: + """Apply a settings update. + + Settings are stored but not applied to the active connection. + + Args: + update: A :class:`STTSettings` (or ``AssemblyAISTTSettings``) delta. + + Returns: + Dict mapping changed field names to their previous values. + """ + changed = await super()._update_settings(update) + + if not changed: + return changed + + # TODO: someday we could reconnect here to apply updated settings. + # Code might look something like the below: + # # Re-apply manual turn mode config if vad_force_turn_endpoint is active + # # and connection_params were updated. + # if self._vad_force_turn_endpoint and "connection_params" in changed: + # self._settings.connection_params = self._configure_manual_turn_mode( + # self._settings.connection_params + # ) + # await self._disconnect() + # await self._connect() + + self._warn_unhandled_updated_settings(changed) + + return changed + async def start(self, frame: StartFrame): """Start the speech-to-text service. @@ -239,7 +291,7 @@ class AssemblyAISTTService(WebsocketSTTService): def _build_ws_url(self) -> str: """Build WebSocket URL with query parameters using urllib.parse.urlencode.""" params = {} - for k, v in self._connection_params.model_dump().items(): + for k, v in self._settings.connection_params.model_dump().items(): if v is not None: if k == "keyterms_prompt": params[k] = json.dumps(v) @@ -415,18 +467,18 @@ class AssemblyAISTTService(WebsocketSTTService): if not message.transcript: return if message.end_of_turn and ( - not self._connection_params.formatted_finals or message.turn_is_formatted + not self._settings.connection_params.formatted_finals or message.turn_is_formatted ): await self.push_frame( TranscriptionFrame( message.transcript, self._user_id, time_now_iso8601(), - self._language, + self._settings.language, message, ) ) - await self._trace_transcription(message.transcript, True, self._language) + await self._trace_transcription(message.transcript, True, self._settings.language) await self.stop_processing_metrics() else: await self.push_frame( @@ -434,7 +486,7 @@ class AssemblyAISTTService(WebsocketSTTService): message.transcript, self._user_id, time_now_iso8601(), - self._language, + self._settings.language, message, ) ) diff --git a/src/pipecat/services/asyncai/tts.py b/src/pipecat/services/asyncai/tts.py index 69ed90ca1..289c327b0 100644 --- a/src/pipecat/services/asyncai/tts.py +++ b/src/pipecat/services/asyncai/tts.py @@ -9,7 +9,8 @@ import asyncio import base64 import json -from typing import AsyncGenerator, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, ClassVar, Dict, Mapping, Optional import aiohttp from loguru import logger @@ -27,6 +28,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven from pipecat.services.tts_service import AudioContextTTSService, TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -72,12 +74,40 @@ def language_to_async_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +@dataclass +class AsyncAITTSSettings(TTSSettings): + """Settings for Async AI TTS services. + + Parameters: + output_container: Audio container format (e.g. "raw"). + output_encoding: Audio encoding format (e.g. "pcm_s16le"). + output_sample_rate: Audio sample rate in Hz. + """ + + output_container: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + output_encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + output_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + @classmethod + def from_mapping(cls, settings: Mapping[str, Any]) -> "AsyncAITTSSettings": + """Construct settings from a plain dict, destructuring legacy nested ``output_format``.""" + flat = dict(settings) + nested = flat.pop("output_format", None) + if isinstance(nested, dict): + flat.setdefault("output_container", nested.get("container")) + flat.setdefault("output_encoding", nested.get("encoding")) + flat.setdefault("output_sample_rate", nested.get("sample_rate")) + return super().from_mapping(flat) + + class AsyncAITTSService(AudioContextTTSService): """Async TTS service with WebSocket streaming. Provides text-to-speech using Async's streaming WebSocket API. """ + _settings: AsyncAITTSSettings + class InputParams(BaseModel): """Input parameters for Async TTS configuration. @@ -131,23 +161,35 @@ class AsyncAITTSService(AudioContextTTSService): self._api_key = api_key self._api_version = version self._url = url - self._settings = { - "output_format": { - "container": container, - "encoding": encoding, - "sample_rate": 0, - }, - "language": self.language_to_service_language(params.language) + self._settings = AsyncAITTSSettings( + model=model, + voice=voice_id, + output_container=container, + output_encoding=encoding, + output_sample_rate=0, + language=self.language_to_service_language(params.language) if params.language else None, - } - - self.set_model_name(model) - self.set_voice(voice_id) + ) + self._sync_model_name_to_metrics() self._receive_task = None self._keepalive_task = None + async def _update_settings(self, update: TTSSettings) -> dict[str, Any]: + """Apply a settings update. + + Settings are stored but not applied to the active connection. + """ + changed = await super()._update_settings(update) + + if not changed: + return changed + + self._warn_unhandled_updated_settings(changed) + + return changed + def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -178,7 +220,7 @@ class AsyncAITTSService(AudioContextTTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["output_format"]["sample_rate"] = self.sample_rate + self._settings.output_sample_rate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): @@ -232,10 +274,14 @@ class AsyncAITTSService(AudioContextTTSService): f"{self._url}?api_key={self._api_key}&version={self._api_version}" ) init_msg = { - "model_id": self._model_name, - "voice": {"mode": "id", "id": self._voice_id}, - "output_format": self._settings["output_format"], - "language": self._settings["language"], + "model_id": self._settings.model, + "voice": {"mode": "id", "id": self._settings.voice}, + "output_format": { + "container": self._settings.output_container, + "encoding": self._settings.output_encoding, + "sample_rate": self._settings.output_sample_rate, + }, + "language": self._settings.language, } await self._get_websocket().send(json.dumps(init_msg)) @@ -404,6 +450,8 @@ class AsyncAIHttpTTSService(TTSService): connection is not required or desired. """ + _settings: AsyncAITTSSettings + class InputParams(BaseModel): """Input parameters for Async API. @@ -450,18 +498,17 @@ class AsyncAIHttpTTSService(TTSService): self._api_key = api_key self._base_url = url self._api_version = version - self._settings = { - "output_format": { - "container": container, - "encoding": encoding, - "sample_rate": 0, - }, - "language": self.language_to_service_language(params.language) + self._settings = AsyncAITTSSettings( + model=model, + voice=voice_id, + output_container=container, + output_encoding=encoding, + output_sample_rate=0, + language=self.language_to_service_language(params.language) if params.language else None, - } - self.set_voice(voice_id) - self.set_model_name(model) + ) + self._sync_model_name_to_metrics() self._session = aiohttp_session @@ -491,7 +538,7 @@ class AsyncAIHttpTTSService(TTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["output_format"]["sample_rate"] = self.sample_rate + self._settings.output_sample_rate = self.sample_rate @traced_tts async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]: @@ -507,14 +554,18 @@ class AsyncAIHttpTTSService(TTSService): logger.debug(f"{self}: Generating TTS [{text}]") try: - voice_config = {"mode": "id", "id": self._voice_id} + voice_config = {"mode": "id", "id": self._settings.voice} await self.start_ttfb_metrics() payload = { - "model_id": self._model_name, + "model_id": self._settings.model, "transcript": text, "voice": voice_config, - "output_format": self._settings["output_format"], - "language": self._settings["language"], + "output_format": { + "container": self._settings.output_container, + "encoding": self._settings.output_encoding, + "sample_rate": self._settings.output_sample_rate, + }, + "language": self._settings.language, } yield TTSStartedFrame(context_id=context_id) headers = { diff --git a/src/pipecat/services/aws/llm.py b/src/pipecat/services/aws/llm.py index 1778ae74e..de994463b 100644 --- a/src/pipecat/services/aws/llm.py +++ b/src/pipecat/services/aws/llm.py @@ -18,8 +18,8 @@ import io import json import os import re -from dataclasses import dataclass -from typing import Any, Dict, List, Optional +from dataclasses import dataclass, field +from typing import Any, ClassVar, Dict, List, Optional from loguru import logger from PIL import Image @@ -40,7 +40,6 @@ from pipecat.frames.frames import ( LLMFullResponseStartFrame, LLMMessagesFrame, LLMTextFrame, - LLMUpdateSettingsFrame, UserImageRawFrame, ) from pipecat.metrics.metrics import LLMTokenUsage @@ -57,6 +56,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import LLMService +from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven from pipecat.utils.tracing.service_decorators import traced_llm try: @@ -71,6 +71,21 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class AWSBedrockLLMSettings(LLMSettings): + """Settings for AWS Bedrock LLM services. + + Parameters: + latency: Performance mode - "standard" or "optimized". + additional_model_request_fields: Additional model-specific parameters. + """ + + latency: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + additional_model_request_fields: Dict[str, Any] | _NotGiven = field( + default_factory=lambda: NOT_GIVEN + ) + + @dataclass class AWSBedrockContextAggregatorPair: """Container for AWS Bedrock context aggregators. @@ -730,6 +745,8 @@ class AWSBedrockLLMService(LLMService): vision capabilities. """ + _settings: AWSBedrockLLMSettings + # Overriding the default adapter to use the Anthropic one. adapter_class = AWSBedrockLLMAdapter @@ -803,18 +820,19 @@ class AWSBedrockLLMService(LLMService): "config": client_config, } - self.set_model_name(model) self._retry_timeout_secs = retry_timeout_secs self._retry_on_timeout = retry_on_timeout - self._settings = { - "max_tokens": params.max_tokens, - "temperature": params.temperature, - "top_p": params.top_p, - "latency": params.latency, - "additional_model_request_fields": params.additional_model_request_fields + self._settings = AWSBedrockLLMSettings( + model=model, + max_tokens=params.max_tokens, + temperature=params.temperature, + top_p=params.top_p, + latency=params.latency, + additional_model_request_fields=params.additional_model_request_fields if isinstance(params.additional_model_request_fields, dict) else {}, - } + ) + self._sync_model_name_to_metrics() logger.info(f"Using AWS Bedrock model: {model}") @@ -836,12 +854,12 @@ class AWSBedrockLLMService(LLMService): Dictionary containing only the inference parameters that are not None. """ inference_config = {} - if self._settings["max_tokens"] is not None: - inference_config["maxTokens"] = self._settings["max_tokens"] - if self._settings["temperature"] is not None: - inference_config["temperature"] = self._settings["temperature"] - if self._settings["top_p"] is not None: - inference_config["topP"] = self._settings["top_p"] + if self._settings.max_tokens is not None: + inference_config["maxTokens"] = self._settings.max_tokens + if self._settings.temperature is not None: + inference_config["temperature"] = self._settings.temperature + if self._settings.top_p is not None: + inference_config["topP"] = self._settings.top_p return inference_config async def run_inference( @@ -877,9 +895,9 @@ class AWSBedrockLLMService(LLMService): inference_config["maxTokens"] = max_tokens request_params = { - "modelId": self.model_name, + "modelId": self._settings.model, "messages": messages, - "additionalModelRequestFields": self._settings["additional_model_request_fields"], + "additionalModelRequestFields": self._settings.additional_model_request_fields, } if inference_config: @@ -1034,9 +1052,9 @@ class AWSBedrockLLMService(LLMService): # Prepare request parameters request_params = { - "modelId": self.model_name, + "modelId": self._settings.model, "messages": messages, - "additionalModelRequestFields": self._settings["additional_model_request_fields"], + "additionalModelRequestFields": self._settings.additional_model_request_fields, } # Only add inference config if it has parameters @@ -1081,8 +1099,8 @@ class AWSBedrockLLMService(LLMService): request_params["toolConfig"] = tool_config # Add performance config if latency is specified - if self._settings["latency"] in ["standard", "optimized"]: - request_params["performanceConfig"] = {"latency": self._settings["latency"]} + if self._settings.latency in ["standard", "optimized"]: + request_params["performanceConfig"] = {"latency": self._settings.latency} # Log request params with messages redacted for logging if isinstance(context, LLMContext): @@ -1207,8 +1225,6 @@ class AWSBedrockLLMService(LLMService): # NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal # LLMContext with it context = AWSBedrockLLMContext.from_messages(frame.messages) - elif isinstance(frame, LLMUpdateSettingsFrame): - await self._update_settings(frame.settings) else: await self.push_frame(frame, direction) diff --git a/src/pipecat/services/aws/nova_sonic/llm.py b/src/pipecat/services/aws/nova_sonic/llm.py index 05baba2bd..eba5cc21b 100644 --- a/src/pipecat/services/aws/nova_sonic/llm.py +++ b/src/pipecat/services/aws/nova_sonic/llm.py @@ -16,7 +16,7 @@ import json import time import uuid import wave -from dataclasses import dataclass +from dataclasses import dataclass, field from enum import Enum from importlib.resources import files from typing import Any, List, Optional @@ -60,6 +60,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import LLMService +from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven from pipecat.utils.time import time_now_iso8601 try: @@ -185,6 +186,20 @@ class Params(BaseModel): endpointing_sensitivity: Optional[str] = Field(default=None) +@dataclass +class AWSNovaSonicLLMSettings(LLMSettings): + """Settings for AWS Nova Sonic LLM service. + + Parameters: + voice_id: Voice for speech synthesis. + endpointing_sensitivity: Controls how quickly Nova Sonic decides the + user has stopped speaking. Can be "LOW", "MEDIUM", or "HIGH". + """ + + voice_id: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + endpointing_sensitivity: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class AWSNovaSonicLLMService(LLMService): """AWS Nova Sonic speech-to-speech LLM service. @@ -192,6 +207,8 @@ class AWSNovaSonicLLMService(LLMService): and function calling capabilities using AWS Nova Sonic model. """ + _settings: AWSNovaSonicLLMSettings + # Override the default adapter to use the AWSNovaSonicLLMAdapter one adapter_class = AWSNovaSonicLLMAdapter @@ -242,23 +259,38 @@ class AWSNovaSonicLLMService(LLMService): self._access_key_id = access_key_id self._session_token = session_token self._region = region - self._model = model self._client: Optional[BedrockRuntimeClient] = None - self._voice_id = voice_id - self._params = params or Params() + params = params or Params() + self._settings = AWSNovaSonicLLMSettings( + model=model, + voice_id=voice_id, + temperature=params.temperature, + max_tokens=params.max_tokens, + top_p=params.top_p, + endpointing_sensitivity=params.endpointing_sensitivity, + ) + self._sync_model_name_to_metrics() + + # Audio I/O config (hardware settings, not runtime-tunable) + self._input_sample_rate = params.input_sample_rate + self._input_sample_size = params.input_sample_size + self._input_channel_count = params.input_channel_count + self._output_sample_rate = params.output_sample_rate + self._output_sample_size = params.output_sample_size + self._output_channel_count = params.output_channel_count self._system_instruction = system_instruction self._tools = tools # Validate endpointing_sensitivity parameter if ( - self._params.endpointing_sensitivity + self._settings.endpointing_sensitivity and not self._is_endpointing_sensitivity_supported() ): logger.warning( f"endpointing_sensitivity is not supported for model '{model}' and will be ignored. " "This parameter is only supported starting with Nova 2 Sonic (amazon.nova-2-sonic-v1:0)." ) - self._params.endpointing_sensitivity = None + self._settings.endpointing_sensitivity = None if not send_transcription_frames: import warnings @@ -302,6 +334,29 @@ class AWSNovaSonicLLMService(LLMService): with wave.open(file_path.open("rb"), "rb") as wav_file: self._assistant_response_trigger_audio = wav_file.readframes(wav_file.getnframes()) + # + # settings + # + + async def _update_settings(self, update: AWSNovaSonicLLMSettings) -> dict[str, Any]: + """Apply a settings update. + + Settings are stored but not applied to the active connection. + """ + changed = await super()._update_settings(update) + + if not changed: + return changed + + # TODO: someday we could reconnect here to apply updated settings. + # Code might look something like the below: + # await self._disconnect() + # await self._start_connecting() + + self._warn_unhandled_updated_settings(changed) + + return changed + # # standard AIService frame handling # @@ -472,7 +527,7 @@ class AWSNovaSonicLLMService(LLMService): # Start the bidirectional stream self._stream = await self._client.invoke_model_with_bidirectional_stream( - InvokeModelWithBidirectionalStreamOperationInput(model_id=self._model) + InvokeModelWithBidirectionalStreamOperationInput(model_id=self._settings.model) ) # Send session start event @@ -639,7 +694,7 @@ class AWSNovaSonicLLMService(LLMService): def _is_first_generation_sonic_model(self) -> bool: # Nova Sonic (the older model) is identified by "amazon.nova-sonic-v1:0" - return self._model == "amazon.nova-sonic-v1:0" + return self._settings.model == "amazon.nova-sonic-v1:0" def _is_endpointing_sensitivity_supported(self) -> bool: # endpointing_sensitivity is only supported with Nova 2 Sonic (and, @@ -658,9 +713,9 @@ class AWSNovaSonicLLMService(LLMService): turn_detection_config = ( f""", "turnDetectionConfiguration": {{ - "endpointingSensitivity": "{self._params.endpointing_sensitivity}" + "endpointingSensitivity": "{self._settings.endpointing_sensitivity}" }}""" - if self._params.endpointing_sensitivity + if self._settings.endpointing_sensitivity else "" ) @@ -669,9 +724,9 @@ class AWSNovaSonicLLMService(LLMService): "event": {{ "sessionStart": {{ "inferenceConfiguration": {{ - "maxTokens": {self._params.max_tokens}, - "topP": {self._params.top_p}, - "temperature": {self._params.temperature} + "maxTokens": {self._settings.max_tokens}, + "topP": {self._settings.top_p}, + "temperature": {self._settings.temperature} }}{turn_detection_config} }} }} @@ -706,10 +761,10 @@ class AWSNovaSonicLLMService(LLMService): }}, "audioOutputConfiguration": {{ "mediaType": "audio/lpcm", - "sampleRateHertz": {self._params.output_sample_rate}, - "sampleSizeBits": {self._params.output_sample_size}, - "channelCount": {self._params.output_channel_count}, - "voiceId": "{self._voice_id}", + "sampleRateHertz": {self._output_sample_rate}, + "sampleSizeBits": {self._output_sample_size}, + "channelCount": {self._output_channel_count}, + "voiceId": "{self._settings.voice_id}", "encoding": "base64", "audioType": "SPEECH" }}{tools_config} @@ -734,9 +789,9 @@ class AWSNovaSonicLLMService(LLMService): "role": "USER", "audioInputConfiguration": {{ "mediaType": "audio/lpcm", - "sampleRateHertz": {self._params.input_sample_rate}, - "sampleSizeBits": {self._params.input_sample_size}, - "channelCount": {self._params.input_channel_count}, + "sampleRateHertz": {self._input_sample_rate}, + "sampleSizeBits": {self._input_sample_size}, + "channelCount": {self._input_channel_count}, "audioType": "SPEECH", "encoding": "base64" }} @@ -1019,8 +1074,8 @@ class AWSNovaSonicLLMService(LLMService): audio = base64.b64decode(audio_content) frame = TTSAudioRawFrame( audio=audio, - sample_rate=self._params.output_sample_rate, - num_channels=self._params.output_channel_count, + sample_rate=self._output_sample_rate, + num_channels=self._output_channel_count, ) await self.push_frame(frame) @@ -1304,7 +1359,7 @@ class AWSNovaSonicLLMService(LLMService): """ if not self._is_assistant_response_trigger_needed(): logger.warning( - f"Assistant response trigger not needed for model '{self._model}'; skipping. " + f"Assistant response trigger not needed for model '{self._settings.model}'; skipping. " "An LLMRunFrame() should be sufficient to prompt the assistant to respond, " "assuming the context ends in a user message." ) @@ -1332,9 +1387,9 @@ class AWSNovaSonicLLMService(LLMService): chunk_duration = 0.02 # what we might get from InputAudioRawFrame chunk_size = int( chunk_duration - * self._params.input_sample_rate - * self._params.input_channel_count - * (self._params.input_sample_size / 8) + * self._input_sample_rate + * self._input_channel_count + * (self._input_sample_size / 8) ) # e.g. 0.02 seconds of 16-bit (2-byte) PCM mono audio at 16kHz is 640 bytes # Lead with a bit of blank audio, if needed. diff --git a/src/pipecat/services/aws/stt.py b/src/pipecat/services/aws/stt.py index f78bc4d4b..09552ecfc 100644 --- a/src/pipecat/services/aws/stt.py +++ b/src/pipecat/services/aws/stt.py @@ -14,7 +14,8 @@ import json import os import random import string -from typing import AsyncGenerator, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, Optional from loguru import logger @@ -28,6 +29,7 @@ from pipecat.frames.frames import ( TranscriptionFrame, ) from pipecat.services.aws.utils import build_event_message, decode_event, get_presigned_url +from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven from pipecat.services.stt_latency import AWS_TRANSCRIBE_TTFS_P99 from pipecat.services.stt_service import WebsocketSTTService from pipecat.transcriptions.language import Language, resolve_language @@ -43,6 +45,25 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class AWSTranscribeSTTSettings(STTSettings): + """Settings for the AWS Transcribe STT service. + + Parameters: + sample_rate: Audio sample rate in Hz (8000 or 16000). + media_encoding: Audio encoding format (e.g. "linear16"). + number_of_channels: Number of audio channels. + show_speaker_label: Whether to show speaker labels. + enable_channel_identification: Whether to enable channel identification. + """ + + sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + media_encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + number_of_channels: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + show_speaker_label: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + enable_channel_identification: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class AWSTranscribeSTTService(WebsocketSTTService): """AWS Transcribe Speech-to-Text service using WebSocket streaming. @@ -51,6 +72,8 @@ class AWSTranscribeSTTService(WebsocketSTTService): final transcription results. """ + _settings: AWSTranscribeSTTSettings + def __init__( self, *, @@ -78,21 +101,21 @@ class AWSTranscribeSTTService(WebsocketSTTService): """ super().__init__(ttfs_p99_latency=ttfs_p99_latency, **kwargs) - self._settings = { - "sample_rate": sample_rate, - "language": language, - "media_encoding": "linear16", # AWS expects raw PCM - "number_of_channels": 1, - "show_speaker_label": False, - "enable_channel_identification": False, - } + self._settings = AWSTranscribeSTTSettings( + language=self.language_to_service_language(language) or "en-US", + sample_rate=sample_rate, + media_encoding="linear16", + number_of_channels=1, + show_speaker_label=False, + enable_channel_identification=False, + ) # Validate sample rate - AWS Transcribe only supports 8000 Hz or 16000 Hz if sample_rate not in [8000, 16000]: logger.warning( f"AWS Transcribe only supports 8000 Hz or 16000 Hz sample rates. Converting from {sample_rate} Hz to 16000 Hz." ) - self._settings["sample_rate"] = 16000 + self._settings.sample_rate = 16000 self._credentials = { "aws_access_key_id": aws_access_key_id or os.getenv("AWS_ACCESS_KEY_ID"), @@ -117,6 +140,26 @@ class AWSTranscribeSTTService(WebsocketSTTService): } return encoding_map.get(encoding, encoding) + async def _update_settings(self, update: STTSettings) -> dict[str, Any]: + """Apply a settings update. + + Settings are stored but not applied to the active connection. + """ + changed = await super()._update_settings(update) + + if not changed: + return changed + + # TODO: someday we could reconnect here to apply updated settings. + # Code might look something like the below: + # if changed and self._websocket: + # await self._disconnect() + # await self._connect() + + self._warn_unhandled_updated_settings(changed) + + return changed + async def start(self, frame: StartFrame): """Initialize the connection when the service starts. @@ -208,9 +251,9 @@ class AWSTranscribeSTTService(WebsocketSTTService): logger.debug("Connecting to AWS Transcribe WebSocket") - language_code = self.language_to_service_language(Language(self._settings["language"])) + language_code = self._settings.language if not language_code: - raise ValueError(f"Unsupported language: {self._settings['language']}") + raise ValueError(f"Unsupported language: {language_code}") # Generate random websocket key websocket_key = "".join( @@ -237,14 +280,14 @@ class AWSTranscribeSTTService(WebsocketSTTService): }, language_code=language_code, media_encoding=self.get_service_encoding( - self._settings["media_encoding"] + self._settings.media_encoding ), # Convert to AWS format - sample_rate=self._settings["sample_rate"], - number_of_channels=self._settings["number_of_channels"], + sample_rate=self._settings.sample_rate, + number_of_channels=self._settings.number_of_channels, enable_partial_results_stabilization=True, partial_results_stability="high", - show_speaker_label=self._settings["show_speaker_label"], - enable_channel_identification=self._settings["enable_channel_identification"], + show_speaker_label=self._settings.show_speaker_label, + enable_channel_identification=self._settings.enable_channel_identification, ) logger.debug(f"{self} Connecting to WebSocket with URL: {presigned_url[:100]}...") @@ -479,14 +522,14 @@ class AWSTranscribeSTTService(WebsocketSTTService): transcript, self._user_id, time_now_iso8601(), - self._settings["language"], + self._settings.language, result=result, ) ) await self._handle_transcription( transcript, is_final, - self._settings["language"], + self._settings.language, ) await self.stop_processing_metrics() else: @@ -495,7 +538,7 @@ class AWSTranscribeSTTService(WebsocketSTTService): transcript, self._user_id, time_now_iso8601(), - self._settings["language"], + self._settings.language, result=result, ) ) diff --git a/src/pipecat/services/aws/tts.py b/src/pipecat/services/aws/tts.py index b902564d2..4e071465d 100644 --- a/src/pipecat/services/aws/tts.py +++ b/src/pipecat/services/aws/tts.py @@ -11,6 +11,7 @@ supporting multiple languages, voices, and SSML features. """ import os +from dataclasses import dataclass, field from typing import AsyncGenerator, List, Optional from loguru import logger @@ -24,6 +25,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven from pipecat.services.tts_service import TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -121,6 +123,25 @@ def language_to_aws_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +@dataclass +class AWSPollyTTSSettings(TTSSettings): + """Settings for AWS Polly TTS service. + + Parameters: + engine: TTS engine to use ('standard', 'neural', etc.). + pitch: Voice pitch adjustment (for standard engine only). + rate: Speech rate adjustment. + volume: Voice volume adjustment. + lexicon_names: List of pronunciation lexicons to apply. + """ + + engine: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + pitch: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + rate: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + volume: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + lexicon_names: List[str] | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class AWSPollyTTSService(TTSService): """AWS Polly text-to-speech service. @@ -129,6 +150,8 @@ class AWSPollyTTSService(TTSService): options including prosody controls. """ + _settings: AWSPollyTTSSettings + class InputParams(BaseModel): """Input parameters for AWS Polly TTS configuration. @@ -185,21 +208,20 @@ class AWSPollyTTSService(TTSService): } self._aws_session = aioboto3.Session() - self._settings = { - "engine": params.engine, - "language": self.language_to_service_language(params.language) + self._settings = AWSPollyTTSSettings( + voice=voice_id, + engine=params.engine, + language=self.language_to_service_language(params.language) if params.language else "en-US", - "pitch": params.pitch, - "rate": params.rate, - "volume": params.volume, - "lexicon_names": params.lexicon_names, - } + pitch=params.pitch, + rate=params.rate, + volume=params.volume, + lexicon_names=params.lexicon_names, + ) self._resampler = create_stream_resampler() - self.set_voice(voice_id) - def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -222,19 +244,19 @@ class AWSPollyTTSService(TTSService): def _construct_ssml(self, text: str) -> str: ssml = "" - language = self._settings["language"] + language = self._settings.language ssml += f"" prosody_attrs = [] # Prosody tags are only supported for standard and neural engines - if self._settings["engine"] == "standard": - if self._settings["pitch"]: - prosody_attrs.append(f"pitch='{self._settings['pitch']}'") + if self._settings.engine == "standard": + if self._settings.pitch: + prosody_attrs.append(f"pitch='{self._settings.pitch}'") - if self._settings["rate"]: - prosody_attrs.append(f"rate='{self._settings['rate']}'") - if self._settings["volume"]: - prosody_attrs.append(f"volume='{self._settings['volume']}'") + if self._settings.rate: + prosody_attrs.append(f"rate='{self._settings.rate}'") + if self._settings.volume: + prosody_attrs.append(f"volume='{self._settings.volume}'") if prosody_attrs: ssml += f"" @@ -275,11 +297,11 @@ class AWSPollyTTSService(TTSService): "Text": ssml, "TextType": "ssml", "OutputFormat": "pcm", - "VoiceId": self._voice_id, - "Engine": self._settings["engine"], + "VoiceId": self._settings.voice, + "Engine": self._settings.engine, # AWS only supports 8000 and 16000 for PCM. We select 16000. "SampleRate": "16000", - "LexiconNames": self._settings["lexicon_names"], + "LexiconNames": self._settings.lexicon_names, } # Filter out None values diff --git a/src/pipecat/services/azure/image.py b/src/pipecat/services/azure/image.py index 2bddf6c43..f5ce4a9f1 100644 --- a/src/pipecat/services/azure/image.py +++ b/src/pipecat/services/azure/image.py @@ -54,7 +54,8 @@ class AzureImageGenServiceREST(ImageGenService): self._api_key = api_key self._azure_endpoint = endpoint self._api_version = api_version - self.set_model_name(model) + self._settings.model = model + self._sync_model_name_to_metrics() self._image_size = image_size self._aiohttp_session = aiohttp_session diff --git a/src/pipecat/services/azure/stt.py b/src/pipecat/services/azure/stt.py index 1bc7ec70a..d161b3829 100644 --- a/src/pipecat/services/azure/stt.py +++ b/src/pipecat/services/azure/stt.py @@ -11,7 +11,8 @@ Speech SDK for real-time audio transcription. """ import asyncio -from typing import AsyncGenerator, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, Optional from loguru import logger @@ -25,6 +26,7 @@ from pipecat.frames.frames import ( TranscriptionFrame, ) from pipecat.services.azure.common import language_to_azure_language +from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven from pipecat.services.stt_latency import AZURE_TTFS_P99 from pipecat.services.stt_service import STTService from pipecat.transcriptions.language import Language @@ -48,6 +50,19 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class AzureSTTSettings(STTSettings): + """Settings for the Azure STT service. + + Parameters: + region: Azure region for the Speech service. + sample_rate: Audio sample rate in Hz. + """ + + region: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + sample_rate: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class AzureSTTService(STTService): """Azure Speech-to-Text service for real-time audio transcription. @@ -56,6 +71,8 @@ class AzureSTTService(STTService): provides real-time transcription results with timing information. """ + _settings: AzureSTTSettings + def __init__( self, *, @@ -92,11 +109,11 @@ class AzureSTTService(STTService): self._audio_stream = None self._speech_recognizer = None - self._settings = { - "region": region, - "language": language_to_azure_language(language), - "sample_rate": sample_rate, - } + self._settings = AzureSTTSettings( + region=region, + language=language_to_azure_language(language), + sample_rate=sample_rate, + ) def can_generate_metrics(self) -> bool: """Check if this service can generate performance metrics. @@ -106,6 +123,38 @@ class AzureSTTService(STTService): """ return True + def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to Azure service-specific language code. + + Args: + language: The language to convert. + + Returns: + The Azure-specific language identifier, or None if not supported. + """ + return language_to_azure_language(language) + + async def _update_settings(self, update: STTSettings) -> dict[str, Any]: + """Apply a settings update. + + Settings are stored but not applied to the active recognizer. + """ + changed = await super()._update_settings(update) + + # TODO: someday we could reconnect here to apply updated settings. + # Code might look something like the below: + # if "language" in changed: + # self._speech_config.speech_recognition_language = self._settings.language + # if self._speech_recognizer: + # # Requires refactoring to set up and tear down recognizer, as + # # language is applied at recognizer initialization + # await self._disconnect() + # await self._connect() + + self._warn_unhandled_updated_settings(changed) + + return changed + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: """Process audio data for speech-to-text conversion. @@ -198,7 +247,7 @@ class AzureSTTService(STTService): def _on_handle_recognized(self, event): if event.result.reason == ResultReason.RecognizedSpeech and len(event.result.text) > 0: - language = getattr(event.result, "language", None) or self._settings.get("language") + language = getattr(event.result, "language", None) or self._settings.language frame = TranscriptionFrame( event.result.text, self._user_id, @@ -213,7 +262,7 @@ class AzureSTTService(STTService): def _on_handle_recognizing(self, event): if event.result.reason == ResultReason.RecognizingSpeech and len(event.result.text) > 0: - language = getattr(event.result, "language", None) or self._settings.get("language") + language = getattr(event.result, "language", None) or self._settings.language frame = InterimTranscriptionFrame( event.result.text, self._user_id, diff --git a/src/pipecat/services/azure/tts.py b/src/pipecat/services/azure/tts.py index 7d4aa0253..f1bf9d400 100644 --- a/src/pipecat/services/azure/tts.py +++ b/src/pipecat/services/azure/tts.py @@ -7,6 +7,7 @@ """Azure Cognitive Services Text-to-Speech service implementations.""" import asyncio +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional from loguru import logger @@ -25,6 +26,7 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.azure.common import language_to_azure_language +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven from pipecat.services.tts_service import TTSService, WordTTSService from pipecat.transcriptions.language import Language from pipecat.utils.tracing.service_decorators import traced_tts @@ -65,6 +67,31 @@ def sample_rate_to_output_format(sample_rate: int) -> SpeechSynthesisOutputForma return sample_rate_map.get(sample_rate, SpeechSynthesisOutputFormat.Raw24Khz16BitMonoPcm) +@dataclass +class AzureTTSSettings(TTSSettings): + """Settings for Azure TTS services. + + Parameters: + emphasis: Emphasis level for speech ("strong", "moderate", "reduced"). + language: Language for synthesis. Defaults to English (US). + pitch: Voice pitch adjustment (e.g., "+10%", "-5Hz", "high"). + rate: Speech rate adjustment (e.g., "1.0", "1.25", "slow", "fast"). + role: Voice role for expression (e.g., "YoungAdultFemale"). + style: Speaking style (e.g., "cheerful", "sad", "excited"). + style_degree: Intensity of the speaking style (0.01 to 2.0). + volume: Volume level (e.g., "+20%", "loud", "x-soft"). + """ + + emphasis: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + language: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + pitch: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + rate: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + role: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + style: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + style_degree: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + volume: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class AzureBaseTTSService: """Base mixin class for Azure Cognitive Services text-to-speech implementations. @@ -73,6 +100,8 @@ class AzureBaseTTSService: This is a mixin class and should be used alongside TTSService or its subclasses. """ + _settings: AzureTTSSettings + # Define SSML escape mappings based on SSML reserved characters # See - https://learn.microsoft.com/en-us/azure/ai-services/speech-service/speech-synthesis-markup-structure SSML_ESCAPE_CHARS = { @@ -126,22 +155,22 @@ class AzureBaseTTSService: """ params = params or AzureBaseTTSService.InputParams() - self._settings = { - "emphasis": params.emphasis, - "language": self.language_to_service_language(params.language) + self._settings = AzureTTSSettings( + emphasis=params.emphasis, + language=self.language_to_service_language(params.language) if params.language else "en-US", - "pitch": params.pitch, - "rate": params.rate, - "role": params.role, - "style": params.style, - "style_degree": params.style_degree, - "volume": params.volume, - } + pitch=params.pitch, + rate=params.rate, + role=params.role, + style=params.style, + style_degree=params.style_degree, + voice=voice, + volume=params.volume, + ) self._api_key = api_key self._region = region - self._voice_id = voice self._speech_synthesizer = None def language_to_service_language(self, language: Language) -> Optional[str]: @@ -156,7 +185,7 @@ class AzureBaseTTSService: return language_to_azure_language(language) def _construct_ssml(self, text: str) -> str: - language = self._settings["language"] + language = self._settings.language # Escape special characters escaped_text = self._escape_text(text) @@ -165,42 +194,42 @@ class AzureBaseTTSService: f"" - f"" + f"" "" ) - if self._settings["style"]: - ssml += f"" - if self._settings["emphasis"]: - ssml += f"" + if self._settings.emphasis: + ssml += f"" ssml += escaped_text - if self._settings["emphasis"]: + if self._settings.emphasis: ssml += "" if prosody_attrs: ssml += "" - if self._settings["style"]: + if self._settings.style: ssml += "" ssml += "" @@ -314,7 +343,7 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService): subscription=self._api_key, region=self._region, ) - self._speech_config.speech_synthesis_language = self._settings["language"] + self._speech_config.speech_synthesis_language = self._settings.language self._speech_config.set_speech_synthesis_output_format( sample_rate_to_output_format(self.sample_rate) ) @@ -364,7 +393,7 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService): Returns: True if the language is CJK, False otherwise. """ - language = self._settings.get("language", "").lower() + language = (self._settings.language if self._settings.language else "").lower() # Check if language starts with CJK language codes return language.startswith(("zh", "ja", "ko", "cmn", "yue", "wuu")) @@ -735,7 +764,7 @@ class AzureHttpTTSService(TTSService, AzureBaseTTSService): subscription=self._api_key, region=self._region, ) - self._speech_config.speech_synthesis_language = self._settings["language"] + self._speech_config.speech_synthesis_language = self._settings.language self._speech_config.set_speech_synthesis_output_format( sample_rate_to_output_format(self.sample_rate) ) diff --git a/src/pipecat/services/camb/tts.py b/src/pipecat/services/camb/tts.py index def57d3a0..a2887df28 100644 --- a/src/pipecat/services/camb/tts.py +++ b/src/pipecat/services/camb/tts.py @@ -16,6 +16,7 @@ Features: - Model-specific sample rates: mars-pro (48kHz), mars-flash (22.05kHz) """ +from dataclasses import dataclass, field from typing import Any, AsyncGenerator, Dict, Optional from camb import StreamTtsOutputConfiguration @@ -31,6 +32,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven from pipecat.services.tts_service import TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -133,6 +135,18 @@ def _get_aligned_audio(buffer: bytes) -> tuple[bytes, bytes]: return buffer[:aligned_size], buffer[aligned_size:] +@dataclass +class CambTTSSettings(TTSSettings): + """Settings for Camb.ai TTS service. + + Parameters: + user_instructions: Custom instructions for mars-instruct model only. + Ignored for other models. Max 1000 characters. + """ + + user_instructions: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class CambTTSService(TTSService): """Camb.ai MARS text-to-speech service using the official SDK. @@ -156,6 +170,8 @@ class CambTTSService(TTSService): ) """ + _settings: CambTTSSettings + class InputParams(BaseModel): """Input parameters for Camb.ai TTS configuration. @@ -212,16 +228,15 @@ class CambTTSService(TTSService): ) # Build settings - self._settings = { - "language": ( + self._settings = CambTTSSettings( + model=model, + voice=voice_id, + language=( self.language_to_service_language(params.language) if params.language else "en-us" ), - "user_instructions": params.user_instructions, - } - - self.set_model_name(model) - self.set_voice(str(voice_id)) - self._voice_id = voice_id + user_instructions=params.user_instructions, + ) + self._sync_model_name_to_metrics() self._client = None @@ -256,7 +271,7 @@ class CambTTSService(TTSService): # Use model-specific sample rate if not explicitly specified if not self._init_sample_rate: - self._sample_rate = MODEL_SAMPLE_RATES.get(self.model_name, 22050) + self._sample_rate = MODEL_SAMPLE_RATES.get(self._settings.model, 22050) @traced_tts async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]: @@ -282,15 +297,15 @@ class CambTTSService(TTSService): # Build SDK parameters tts_kwargs: Dict[str, Any] = { "text": text, - "voice_id": self._voice_id, - "language": self._settings["language"], - "speech_model": self.model_name, + "voice_id": self._settings.voice, + "language": self._settings.language, + "speech_model": self._settings.model, "output_configuration": StreamTtsOutputConfiguration(format="pcm_s16le"), } # Add user instructions if using mars-instruct model - if self._model_name == "mars-instruct" and self._settings.get("user_instructions"): - tts_kwargs["user_instructions"] = self._settings["user_instructions"] + if self._settings.model == "mars-instruct" and self._settings.user_instructions: + tts_kwargs["user_instructions"] = self._settings.user_instructions await self.start_tts_usage_metrics(text) yield TTSStartedFrame(context_id=context_id) diff --git a/src/pipecat/services/cartesia/stt.py b/src/pipecat/services/cartesia/stt.py index c4429226f..6a30f9a53 100644 --- a/src/pipecat/services/cartesia/stt.py +++ b/src/pipecat/services/cartesia/stt.py @@ -12,7 +12,8 @@ the Cartesia Live transcription API for real-time speech recognition. import json import urllib.parse -from typing import AsyncGenerator, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, Optional from loguru import logger @@ -27,6 +28,7 @@ from pipecat.frames.frames import ( VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven from pipecat.services.stt_latency import CARTESIA_TTFS_P99 from pipecat.services.stt_service import WebsocketSTTService from pipecat.transcriptions.language import Language @@ -42,6 +44,17 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class CartesiaSTTSettings(STTSettings): + """Settings for the Cartesia STT service. + + Parameters: + encoding: Audio encoding format (e.g. ``"pcm_s16le"``). + """ + + encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class CartesiaLiveOptions: """Configuration options for Cartesia Live STT service. @@ -136,6 +149,8 @@ class CartesiaSTTService(WebsocketSTTService): See: https://docs.cartesia.ai/api-reference/stt/stt """ + _settings: CartesiaSTTSettings + def __init__( self, *, @@ -181,8 +196,12 @@ class CartesiaSTTService(WebsocketSTTService): k: v for k, v in merged_options.items() if not isinstance(v, str) or v != "None" } - self._settings = merged_options - self.set_model_name(merged_options["model"]) + self._settings = CartesiaSTTSettings( + model=merged_options["model"], + language=merged_options.get("language"), + encoding=merged_options.get("encoding", "pcm_s16le"), + ) + self._sync_model_name_to_metrics() self._api_key = api_key self._base_url = base_url or "api.cartesia.ai" self._receive_task = None @@ -275,13 +294,39 @@ class CartesiaSTTService(WebsocketSTTService): await self._disconnect_websocket() + async def _update_settings(self, update: STTSettings) -> dict[str, Any]: + """Apply a settings update. + + Args: + update: A :class:`STTSettings` (or ``CartesiaSTTSettings``) delta. + + Returns: + Dict mapping changed field names to their previous values. + """ + changed = await super()._update_settings(update) + + # TODO: someday we could reconnect here to apply updated settings. + # Code might look something like the below: + # if changed: + # await self._disconnect() + # await self._connect() + + self._warn_unhandled_updated_settings(changed) + + return changed + async def _connect_websocket(self): try: if self._websocket and self._websocket.state is State.OPEN: return logger.debug("Connecting to Cartesia STT") - params = self._settings + params = { + "model": self._settings.model, + "language": self._settings.language, + "encoding": self._settings.encoding, + "sample_rate": str(self.sample_rate), + } ws_url = f"wss://{self._base_url}/stt/websocket?{urllib.parse.urlencode(params)}" headers = {"Cartesia-Version": "2025-04-16", "X-API-Key": self._api_key} diff --git a/src/pipecat/services/cartesia/tts.py b/src/pipecat/services/cartesia/tts.py index e30acdcd1..cca540c72 100644 --- a/src/pipecat/services/cartesia/tts.py +++ b/src/pipecat/services/cartesia/tts.py @@ -9,8 +9,9 @@ import base64 import json import warnings +from dataclasses import dataclass, field from enum import Enum -from typing import AsyncGenerator, List, Literal, Optional +from typing import Any, AsyncGenerator, ClassVar, Dict, List, Literal, Mapping, Optional from loguru import logger from pydantic import BaseModel, Field @@ -27,6 +28,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven, is_given from pipecat.services.tts_service import AudioContextWordTTSService, TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.text.base_text_aggregator import BaseTextAggregator @@ -191,6 +193,42 @@ class CartesiaEmotion(str, Enum): DETERMINED = "determined" +@dataclass +class CartesiaTTSSettings(TTSSettings): + """Settings for Cartesia TTS services. + + Parameters: + output_container: Audio container format (e.g. "raw"). + output_encoding: Audio encoding format (e.g. "pcm_s16le"). + output_sample_rate: Audio sample rate in Hz. + speed: Voice speed control for non-Sonic-3 models (literal values). + emotion: List of emotion controls for non-Sonic-3 models. + generation_config: Generation configuration for Sonic-3 models. Includes volume, + speed (numeric), and emotion (string) parameters. + pronunciation_dict_id: The ID of the pronunciation dictionary to use for + custom pronunciations. + """ + + output_container: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + output_encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + output_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + speed: Literal["slow", "normal", "fast"] | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + emotion: List[str] | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + generation_config: GenerationConfig | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + pronunciation_dict_id: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + @classmethod + def from_mapping(cls, settings: Mapping[str, Any]) -> "CartesiaTTSSettings": + """Construct settings from a plain dict, destructuring legacy nested ``output_format``.""" + flat = dict(settings) + nested = flat.pop("output_format", None) + if isinstance(nested, dict): + flat.setdefault("output_container", nested.get("container")) + flat.setdefault("output_encoding", nested.get("encoding")) + flat.setdefault("output_sample_rate", nested.get("sample_rate")) + return super().from_mapping(flat) + + class CartesiaTTSService(AudioContextWordTTSService): """Cartesia TTS service with WebSocket streaming and word timestamps. @@ -199,6 +237,8 @@ class CartesiaTTSService(AudioContextWordTTSService): customization options including speed and emotion controls. """ + _settings: CartesiaTTSSettings + class InputParams(BaseModel): """Input parameters for Cartesia TTS configuration. @@ -289,22 +329,21 @@ class CartesiaTTSService(AudioContextWordTTSService): self._api_key = api_key self._cartesia_version = cartesia_version self._url = url - self._settings = { - "output_format": { - "container": container, - "encoding": encoding, - "sample_rate": 0, - }, - "language": self.language_to_service_language(params.language) + self._settings = CartesiaTTSSettings( + model=model, + output_container=container, + output_encoding=encoding, + output_sample_rate=0, + language=self.language_to_service_language(params.language) if params.language else None, - "speed": params.speed, - "emotion": params.emotion, - "generation_config": params.generation_config, - "pronunciation_dict_id": params.pronunciation_dict_id, - } - self.set_model_name(model) - self.set_voice(voice_id) + speed=params.speed, + emotion=params.emotion, + generation_config=params.generation_config, + pronunciation_dict_id=params.pronunciation_dict_id, + voice=voice_id, + ) + self._sync_model_name_to_metrics() self._receive_task = None @@ -316,16 +355,6 @@ class CartesiaTTSService(AudioContextWordTTSService): """ return True - async def set_model(self, model: str): - """Set the TTS model. - - Args: - model: The model name to use for synthesis. - """ - self._model_id = model - await super().set_model(model) - logger.info(f"Switching TTS model to: [{model}]") - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Cartesia language format. @@ -390,7 +419,7 @@ class CartesiaTTSService(AudioContextWordTTSService): Returns: List of (word, start_time) tuples processed for the language. """ - current_language = self._settings.get("language") + current_language = self._settings.language # Check if this is a CJK language (if language is None, treat as non-CJK) if current_language and self._is_cjk_language(current_language): @@ -411,9 +440,9 @@ class CartesiaTTSService(AudioContextWordTTSService): ): voice_config = {} voice_config["mode"] = "id" - voice_config["id"] = self._voice_id + voice_config["id"] = self._settings.voice - if self._settings["emotion"]: + if is_given(self._settings.emotion) and self._settings.emotion: with warnings.catch_warnings(): warnings.simplefilter("always") warnings.warn( @@ -422,33 +451,36 @@ class CartesiaTTSService(AudioContextWordTTSService): stacklevel=2, ) voice_config["__experimental_controls"] = {} - if self._settings["emotion"]: - voice_config["__experimental_controls"]["emotion"] = self._settings["emotion"] + voice_config["__experimental_controls"]["emotion"] = self._settings.emotion msg = { "transcript": text, "continue": continue_transcript, "context_id": self.get_active_audio_context_id(), - "model_id": self.model_name, + "model_id": self._settings.model, "voice": voice_config, - "output_format": self._settings["output_format"], + "output_format": { + "container": self._settings.output_container, + "encoding": self._settings.output_encoding, + "sample_rate": self._settings.output_sample_rate, + }, "add_timestamps": add_timestamps, - "use_original_timestamps": False if self.model_name == "sonic" else True, + "use_original_timestamps": False if self._settings.model == "sonic" else True, } - if self._settings["language"]: - msg["language"] = self._settings["language"] + if is_given(self._settings.language) and self._settings.language: + msg["language"] = self._settings.language - if self._settings["speed"]: - msg["speed"] = self._settings["speed"] + if is_given(self._settings.speed) and self._settings.speed: + msg["speed"] = self._settings.speed - if self._settings["generation_config"]: - msg["generation_config"] = self._settings["generation_config"].model_dump( + if is_given(self._settings.generation_config) and self._settings.generation_config: + msg["generation_config"] = self._settings.generation_config.model_dump( exclude_none=True ) - if self._settings["pronunciation_dict_id"]: - msg["pronunciation_dict_id"] = self._settings["pronunciation_dict_id"] + if is_given(self._settings.pronunciation_dict_id) and self._settings.pronunciation_dict_id: + msg["pronunciation_dict_id"] = self._settings.pronunciation_dict_id return json.dumps(msg) @@ -459,7 +491,7 @@ class CartesiaTTSService(AudioContextWordTTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["output_format"]["sample_rate"] = self.sample_rate + self._settings.output_sample_rate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): @@ -636,6 +668,8 @@ class CartesiaHttpTTSService(TTSService): integration is preferred. """ + _settings: CartesiaTTSSettings + class InputParams(BaseModel): """Input parameters for Cartesia HTTP TTS configuration. @@ -693,22 +727,21 @@ class CartesiaHttpTTSService(TTSService): self._api_key = api_key self._base_url = base_url self._cartesia_version = cartesia_version - self._settings = { - "output_format": { - "container": container, - "encoding": encoding, - "sample_rate": 0, - }, - "language": self.language_to_service_language(params.language) + self._settings = CartesiaTTSSettings( + model=model, + voice=voice_id, + output_container=container, + output_encoding=encoding, + output_sample_rate=0, + language=self.language_to_service_language(params.language) if params.language else None, - "speed": params.speed, - "emotion": params.emotion, - "generation_config": params.generation_config, - "pronunciation_dict_id": params.pronunciation_dict_id, - } - self.set_voice(voice_id) - self.set_model_name(model) + speed=params.speed, + emotion=params.emotion, + generation_config=params.generation_config, + pronunciation_dict_id=params.pronunciation_dict_id, + ) + self._sync_model_name_to_metrics() self._client = AsyncCartesia( api_key=api_key, @@ -741,7 +774,7 @@ class CartesiaHttpTTSService(TTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["output_format"]["sample_rate"] = self.sample_rate + self._settings.output_sample_rate = self.sample_rate async def stop(self, frame: EndFrame): """Stop the Cartesia HTTP TTS service. @@ -775,9 +808,9 @@ class CartesiaHttpTTSService(TTSService): logger.debug(f"{self}: Generating TTS [{text}]") try: - voice_config = {"mode": "id", "id": self._voice_id} + voice_config = {"mode": "id", "id": self._settings.voice} - if self._settings["emotion"]: + if is_given(self._settings.emotion) and self._settings.emotion: with warnings.catch_warnings(): warnings.simplefilter("always") warnings.warn( @@ -785,30 +818,39 @@ class CartesiaHttpTTSService(TTSService): DeprecationWarning, stacklevel=2, ) - voice_config["__experimental_controls"] = {"emotion": self._settings["emotion"]} + voice_config["__experimental_controls"] = {"emotion": self._settings.emotion} await self.start_ttfb_metrics() - payload = { - "model_id": self._model_name, - "transcript": text, - "voice": voice_config, - "output_format": self._settings["output_format"], + output_format = { + "container": self._settings.output_container, + "encoding": self._settings.output_encoding, + "sample_rate": self._settings.output_sample_rate, } - if self._settings["language"]: - payload["language"] = self._settings["language"] + payload = { + "model_id": self._settings.model, + "transcript": text, + "voice": voice_config, + "output_format": output_format, + } - if self._settings["speed"]: - payload["speed"] = self._settings["speed"] + if is_given(self._settings.language) and self._settings.language: + payload["language"] = self._settings.language - if self._settings["generation_config"]: - payload["generation_config"] = self._settings["generation_config"].model_dump( + if is_given(self._settings.speed) and self._settings.speed: + payload["speed"] = self._settings.speed + + if is_given(self._settings.generation_config) and self._settings.generation_config: + payload["generation_config"] = self._settings.generation_config.model_dump( exclude_none=True ) - if self._settings["pronunciation_dict_id"]: - payload["pronunciation_dict_id"] = self._settings["pronunciation_dict_id"] + if ( + is_given(self._settings.pronunciation_dict_id) + and self._settings.pronunciation_dict_id + ): + payload["pronunciation_dict_id"] = self._settings.pronunciation_dict_id yield TTSStartedFrame(context_id=context_id) diff --git a/src/pipecat/services/cerebras/llm.py b/src/pipecat/services/cerebras/llm.py index 54ea45ddb..e1ecceef7 100644 --- a/src/pipecat/services/cerebras/llm.py +++ b/src/pipecat/services/cerebras/llm.py @@ -66,16 +66,16 @@ class CerebrasLLMService(OpenAILLMService): Dictionary of parameters for the chat completion request. """ params = { - "model": self.model_name, + "model": self._settings.model, "stream": True, - "seed": self._settings["seed"], - "temperature": self._settings["temperature"], - "top_p": self._settings["top_p"], - "max_completion_tokens": self._settings["max_completion_tokens"], + "seed": self._settings.seed, + "temperature": self._settings.temperature, + "top_p": self._settings.top_p, + "max_completion_tokens": self._settings.max_completion_tokens, } # Messages, tools, tool_choice params.update(params_from_context) - params.update(self._settings["extra"]) + params.update(self._settings.extra) return params diff --git a/src/pipecat/services/deepgram/flux/stt.py b/src/pipecat/services/deepgram/flux/stt.py index 5b091862c..410d272da 100644 --- a/src/pipecat/services/deepgram/flux/stt.py +++ b/src/pipecat/services/deepgram/flux/stt.py @@ -9,6 +9,7 @@ import asyncio import json import time +from dataclasses import dataclass, field from enum import Enum from typing import Any, AsyncGenerator, Dict, Optional from urllib.parse import urlencode @@ -27,6 +28,7 @@ from pipecat.frames.frames import ( UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) +from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven from pipecat.services.stt_service import WebsocketSTTService from pipecat.transcriptions.language import Language from pipecat.utils.time import time_now_iso8601 @@ -67,6 +69,34 @@ class FluxEventType(str, Enum): UPDATE = "Update" +@dataclass +class DeepgramFluxSTTSettings(STTSettings): + """Settings for the Deepgram Flux STT service. + + Parameters: + eager_eot_threshold: EagerEndOfTurn/TurnResumed threshold. Off by default. + Lower values = more aggressive (faster response, more LLM calls). + Higher values = more conservative (slower response, fewer LLM calls). + eot_threshold: End-of-turn confidence required to finish a turn (default 0.7). + eot_timeout_ms: Time in ms after speech to finish a turn regardless of EOT + confidence (default 5000). + keyterm: Keyterms to boost recognition accuracy for specialized terminology. + mip_opt_out: Opt out of the Deepgram Model Improvement Program (default False). + tag: Tags to label requests for identification during usage reporting. + min_confidence: Minimum confidence required to create a TranscriptionFrame. + encoding: Audio encoding format (e.g. ``"linear16"``). + """ + + eager_eot_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + eot_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + eot_timeout_ms: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + keyterm: list | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + mip_opt_out: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + tag: list | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + min_confidence: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class DeepgramFluxSTTService(WebsocketSTTService): """Deepgram Flux speech-to-text service. @@ -89,6 +119,8 @@ class DeepgramFluxSTTService(WebsocketSTTService): ... """ + _settings: DeepgramFluxSTTSettings + class InputParams(BaseModel): """Configuration parameters for Deepgram Flux API. @@ -181,14 +213,23 @@ class DeepgramFluxSTTService(WebsocketSTTService): **kwargs, ) + params = params or DeepgramFluxSTTService.InputParams() + self._settings = DeepgramFluxSTTSettings( + model=model, + language=Language.EN, + encoding=flux_encoding, + eager_eot_threshold=params.eager_eot_threshold, + eot_threshold=params.eot_threshold, + eot_timeout_ms=params.eot_timeout_ms, + keyterm=params.keyterm or [], + mip_opt_out=params.mip_opt_out, + tag=params.tag or [], + min_confidence=params.min_confidence, + ) + self._sync_model_name_to_metrics() self._api_key = api_key self._url = url - self._model = model - self._params = params or DeepgramFluxSTTService.InputParams() self._should_interrupt = should_interrupt - self._flux_encoding = flux_encoding - # This is the currently only supported language - self._language = Language.EN self._websocket_url = None self._receive_task = None # Flux event handlers @@ -343,6 +384,25 @@ class DeepgramFluxSTTService(WebsocketSTTService): """ return True + async def _update_settings(self, update: DeepgramFluxSTTSettings) -> dict[str, Any]: + """Apply a settings update. + + Settings are stored but not applied to the active connection. + """ + changed = await super()._update_settings(update) + + if not changed: + return changed + + # TODO: someday we could reconnect here to apply updated settings. + # Code might look something like the below: + # await self._disconnect() + # await self._connect() + + self._warn_unhandled_updated_settings(changed) + + return changed + async def start(self, frame: StartFrame): """Start the Deepgram Flux STT service. @@ -355,29 +415,29 @@ class DeepgramFluxSTTService(WebsocketSTTService): await super().start(frame) url_params = [ - f"model={self._model}", + f"model={self._settings.model}", f"sample_rate={self.sample_rate}", - f"encoding={self._flux_encoding}", + f"encoding={self._settings.encoding}", ] - if self._params.eager_eot_threshold is not None: - url_params.append(f"eager_eot_threshold={self._params.eager_eot_threshold}") + if self._settings.eager_eot_threshold is not None: + url_params.append(f"eager_eot_threshold={self._settings.eager_eot_threshold}") - if self._params.eot_threshold is not None: - url_params.append(f"eot_threshold={self._params.eot_threshold}") + if self._settings.eot_threshold is not None: + url_params.append(f"eot_threshold={self._settings.eot_threshold}") - if self._params.eot_timeout_ms is not None: - url_params.append(f"eot_timeout_ms={self._params.eot_timeout_ms}") + if self._settings.eot_timeout_ms is not None: + url_params.append(f"eot_timeout_ms={self._settings.eot_timeout_ms}") - if self._params.mip_opt_out is not None: - url_params.append(f"mip_opt_out={str(self._params.mip_opt_out).lower()}") + if self._settings.mip_opt_out is not None: + url_params.append(f"mip_opt_out={str(self._settings.mip_opt_out).lower()}") # Add keyterm parameters (can have multiple) - for keyterm in self._params.keyterm: + for keyterm in self._settings.keyterm: url_params.append(urlencode({"keyterm": keyterm})) # Add tag parameters (can have multiple) - for tag_value in self._params.tag: + for tag_value in self._settings.tag: url_params.append(urlencode({"tag": tag_value})) self._websocket_url = f"{self._url}?{'&'.join(url_params)}" @@ -676,7 +736,7 @@ class DeepgramFluxSTTService(WebsocketSTTService): # Compute the average confidence average_confidence = self._calculate_average_confidence(data) - if not self._params.min_confidence or average_confidence > self._params.min_confidence: + if not self._settings.min_confidence or average_confidence > self._settings.min_confidence: # EndOfTurn means Flux has determined the turn is complete, # so this TranscriptionFrame is always finalized await self.push_frame( @@ -684,7 +744,7 @@ class DeepgramFluxSTTService(WebsocketSTTService): transcript, self._user_id, time_now_iso8601(), - self._language, + self._settings.language, result=data, finalized=True, ) @@ -694,7 +754,7 @@ class DeepgramFluxSTTService(WebsocketSTTService): f"Transcription confidence below min_confidence threshold: {average_confidence}" ) - await self._handle_transcription(transcript, True, self._language) + await self._handle_transcription(transcript, True, self._settings.language) await self.stop_processing_metrics() await self.broadcast_frame(UserStoppedSpeakingFrame) await self._call_event_handler("on_end_of_turn", transcript) @@ -738,7 +798,7 @@ class DeepgramFluxSTTService(WebsocketSTTService): transcript, self._user_id, time_now_iso8601(), - self._language, + self._settings.language, result=data, ) ) diff --git a/src/pipecat/services/deepgram/stt.py b/src/pipecat/services/deepgram/stt.py index c4f72e6c3..28b8a211e 100644 --- a/src/pipecat/services/deepgram/stt.py +++ b/src/pipecat/services/deepgram/stt.py @@ -6,7 +6,8 @@ """Deepgram speech-to-text service implementation.""" -from typing import AsyncGenerator, Dict, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, Dict, Optional from loguru import logger @@ -23,6 +24,7 @@ from pipecat.frames.frames import ( VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven, is_given from pipecat.services.stt_latency import DEEPGRAM_TTFS_P99 from pipecat.services.stt_service import STTService from pipecat.transcriptions.language import Language @@ -45,6 +47,17 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class DeepgramSTTSettings(STTSettings): + """Settings for the Deepgram STT service. + + Parameters: + live_options: Deepgram ``LiveOptions`` for detailed configuration. + """ + + live_options: LiveOptions | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class DeepgramSTTService(STTService): """Deepgram speech-to-text service. @@ -63,6 +76,8 @@ class DeepgramSTTService(STTService): ... """ + _settings: DeepgramSTTSettings + def __init__( self, *, @@ -139,12 +154,18 @@ class DeepgramSTTService(STTService): if "language" in merged_options and isinstance(merged_options["language"], Language): merged_options["language"] = merged_options["language"].value - self.set_model_name(merged_options["model"]) - self._settings = merged_options + merged_live_options = LiveOptions(**merged_options) + self._settings = DeepgramSTTSettings( + model=merged_options.get("model"), + language=merged_options.get("language"), + live_options=merged_live_options, + ) + self._sync_model_name_to_metrics() + self._addons = addons self._should_interrupt = should_interrupt - if merged_options.get("vad_events"): + if merged_live_options.vad_events: import warnings with warnings.catch_warnings(): @@ -175,7 +196,7 @@ class DeepgramSTTService(STTService): Returns: True if VAD events are enabled in the current settings. """ - return self._settings["vad_events"] + return self._settings.live_options.vad_events def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -185,28 +206,48 @@ class DeepgramSTTService(STTService): """ return True - async def set_model(self, model: str): - """Set the Deepgram model and reconnect. + async def _update_settings(self, update: STTSettings) -> dict[str, Any]: + """Apply a settings update, keeping ``live_options`` in sync. - Args: - model: The Deepgram model name to use. + Top-level ``model`` and ``language`` are the source of truth. When + they are given in *update* their values are propagated into + ``live_options``. When only ``live_options`` is given, its ``model`` + and ``language`` are propagated *up* to the top-level fields. + + Any change triggers a WebSocket reconnect. """ - await super().set_model(model) - logger.info(f"Switching STT model to: [{model}]") - self._settings["model"] = model + # Determine which top-level fields are explicitly provided. + model_given = isinstance(update, DeepgramSTTSettings) and is_given( + getattr(update, "model", NOT_GIVEN) + ) + language_given = isinstance(update, DeepgramSTTSettings) and is_given( + getattr(update, "language", NOT_GIVEN) + ) + + changed = await super()._update_settings(update) + + if not changed: + return changed + + # --- Sync model -------------------------------------------------- + if model_given: + # Top-level model wins → push into live_options. + self._settings.live_options.model = self._settings.model + elif "live_options" in changed and self._settings.live_options.model is not None: + # Only live_options was given → pull model up. + self._settings.model = self._settings.live_options.model + self._sync_model_name_to_metrics() + + # --- Sync language ----------------------------------------------- + if language_given: + self._settings.live_options.language = self._settings.language + elif "live_options" in changed and self._settings.live_options.language is not None: + self._settings.language = self._settings.live_options.language + await self._disconnect() await self._connect() - async def set_language(self, language: Language): - """Set the recognition language and reconnect. - - Args: - language: The language to use for speech recognition. - """ - logger.info(f"Switching STT language to: [{language}]") - self._settings["language"] = language - await self._disconnect() - await self._connect() + return changed async def start(self, frame: StartFrame): """Start the Deepgram STT service. @@ -215,7 +256,7 @@ class DeepgramSTTService(STTService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["sample_rate"] = self.sample_rate + self._settings.live_options.sample_rate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): @@ -268,7 +309,9 @@ class DeepgramSTTService(STTService): self._on_utterance_end, ) - if not await self._connection.start(options=self._settings, addons=self._addons): + if not await self._connection.start( + options=self._settings.live_options, addons=self._addons + ): await self.push_error(error_msg=f"Unable to connect to Deepgram") else: headers = { diff --git a/src/pipecat/services/deepgram/stt_sagemaker.py b/src/pipecat/services/deepgram/stt_sagemaker.py index 8fc95b726..64bb2ba8f 100644 --- a/src/pipecat/services/deepgram/stt_sagemaker.py +++ b/src/pipecat/services/deepgram/stt_sagemaker.py @@ -14,7 +14,8 @@ languages, and various Deepgram features. import asyncio import json -from typing import AsyncGenerator, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, Optional from loguru import logger @@ -31,6 +32,7 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.aws.sagemaker.bidi_client import SageMakerBidiClient +from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven, is_given from pipecat.services.stt_latency import DEEPGRAM_SAGEMAKER_TTFS_P99 from pipecat.services.stt_service import STTService from pipecat.transcriptions.language import Language @@ -47,6 +49,17 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class DeepgramSageMakerSTTSettings(STTSettings): + """Settings for the Deepgram SageMaker STT service. + + Parameters: + live_options: Deepgram ``LiveOptions`` for detailed configuration. + """ + + live_options: LiveOptions | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class DeepgramSageMakerSTTService(STTService): """Deepgram speech-to-text service for AWS SageMaker. @@ -75,6 +88,8 @@ class DeepgramSageMakerSTTService(STTService): ) """ + _settings: DeepgramSageMakerSTTSettings + def __init__( self, *, @@ -128,8 +143,13 @@ class DeepgramSageMakerSTTService(STTService): if "language" in merged_options and isinstance(merged_options["language"], Language): merged_options["language"] = merged_options["language"].value - self.set_model_name(merged_options["model"]) - self._settings = merged_options + merged_live_options = LiveOptions(**merged_options) + self._settings = DeepgramSageMakerSTTSettings( + model=merged_options.get("model"), + language=merged_options.get("language"), + live_options=merged_live_options, + ) + self._sync_model_name_to_metrics() self._client: Optional[SageMakerBidiClient] = None self._response_task: Optional[asyncio.Task] = None @@ -143,35 +163,55 @@ class DeepgramSageMakerSTTService(STTService): """ return True - async def set_model(self, model: str): - """Set the Deepgram model and reconnect. + async def _update_settings(self, update: STTSettings) -> dict[str, Any]: + """Apply a settings update, keeping ``live_options`` in sync. - Disconnects from the current session, updates the model setting, and - establishes a new connection with the updated model. + Top-level ``model`` and ``language`` are the source of truth. When + they are given in *update* their values are propagated into + ``live_options``. When only ``live_options`` is given, its ``model`` + and ``language`` are propagated *up* to the top-level fields. - Args: - model: The Deepgram model name to use (e.g., "nova-3"). + Any change triggers a reconnect. """ - await super().set_model(model) - logger.info(f"Switching STT model to: [{model}]") - self._settings["model"] = model - await self._disconnect() - await self._connect() + # Determine which top-level fields are explicitly provided. + model_given = isinstance(update, DeepgramSageMakerSTTSettings) and is_given( + getattr(update, "model", NOT_GIVEN) + ) + language_given = isinstance(update, DeepgramSageMakerSTTSettings) and is_given( + getattr(update, "language", NOT_GIVEN) + ) - async def set_language(self, language: Language): - """Set the recognition language and reconnect. + changed = await super()._update_settings(update) - Disconnects from the current session, updates the language setting, and - establishes a new connection with the updated language. + if not changed: + return changed - Args: - language: The language to use for speech recognition (e.g., Language.EN, - Language.ES). - """ - logger.info(f"Switching STT language to: [{language}]") - self._settings["language"] = language - await self._disconnect() - await self._connect() + # --- Sync model -------------------------------------------------- + if model_given: + # Top-level model wins → push into live_options. + self._settings.live_options.model = self._settings.model + elif "live_options" in changed and self._settings.live_options.model is not None: + # Only live_options was given → pull model up. + self._settings.model = self._settings.live_options.model + self._sync_model_name_to_metrics() + + # --- Sync language ----------------------------------------------- + if language_given: + lang = self._settings.language + if isinstance(lang, Language): + lang = lang.value + self._settings.live_options.language = lang + elif "live_options" in changed and self._settings.live_options.language is not None: + self._settings.language = self._settings.live_options.language + + # TODO: someday we could reconnect here to apply updated settings. + # Code might look something like the below: + # await self._disconnect() + # await self._connect() + + self._warn_unhandled_updated_settings(changed) + + return changed async def start(self, frame: StartFrame): """Start the Deepgram SageMaker STT service. @@ -180,7 +220,7 @@ class DeepgramSageMakerSTTService(STTService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["sample_rate"] = self.sample_rate + self._settings.live_options.sample_rate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): @@ -226,12 +266,12 @@ class DeepgramSageMakerSTTService(STTService): """ logger.debug("Connecting to Deepgram on SageMaker...") - # Update sample rate in settings - self._settings["sample_rate"] = self.sample_rate + # Update sample rate in live_options + self._settings.live_options.sample_rate = self.sample_rate - # Build query string from settings, converting booleans to strings + # Build query string from live_options, converting booleans to strings query_params = {} - for key, value in self._settings.items(): + for key, value in self._settings.live_options.to_dict().items(): if value is not None: # Convert boolean values to lowercase strings for Deepgram API if isinstance(value, bool): diff --git a/src/pipecat/services/deepgram/tts.py b/src/pipecat/services/deepgram/tts.py index 12aba4905..9216d7aa7 100644 --- a/src/pipecat/services/deepgram/tts.py +++ b/src/pipecat/services/deepgram/tts.py @@ -11,7 +11,8 @@ for generating speech from text using various voice models. """ import json -from typing import AsyncGenerator, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, Optional import aiohttp from loguru import logger @@ -29,6 +30,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven from pipecat.services.tts_service import TTSService, WebsocketTTSService from pipecat.utils.tracing.service_decorators import traced_tts @@ -43,6 +45,17 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class DeepgramTTSSettings(TTSSettings): + """Settings for Deepgram TTS service. + + Parameters: + encoding: Audio encoding format (linear16, mulaw, alaw). + """ + + encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class DeepgramTTSService(WebsocketTTSService): """Deepgram WebSocket-based text-to-speech service. @@ -51,6 +64,8 @@ class DeepgramTTSService(WebsocketTTSService): message for conversational AI use cases. """ + _settings: DeepgramTTSSettings + SUPPORTED_ENCODINGS = ("linear16", "mulaw", "alaw") def __init__( @@ -91,10 +106,12 @@ class DeepgramTTSService(WebsocketTTSService): self._api_key = api_key self._base_url = base_url - self._settings = { - "encoding": encoding, - } - self.set_voice(voice) + self._settings = DeepgramTTSSettings( + model=voice, + voice=voice, + encoding=encoding, + ) + self._sync_model_name_to_metrics() self._receive_task = None self._context_id: Optional[str] = None @@ -166,6 +183,28 @@ class DeepgramTTSService(WebsocketTTSService): await self._disconnect_websocket() + async def _update_settings(self, update: TTSSettings) -> dict[str, Any]: + """Apply a settings update. + + Args: + update: A :class:`TTSSettings` (or ``DeepgramTTSSettings``) delta. + + Returns: + Dict mapping changed field names to their previous values. + """ + changed = await super()._update_settings(update) + + # Deepgram uses voice as the model, so keep them in sync for metrics + if "voice" in changed: + self._settings.model = self._settings.voice + self._sync_model_name_to_metrics() + + if changed: + await self._disconnect() + await self._connect() + + return changed + async def _connect_websocket(self): """Connect to Deepgram WebSocket API with configured settings.""" try: @@ -176,8 +215,8 @@ class DeepgramTTSService(WebsocketTTSService): # Build WebSocket URL with query parameters params = [] - params.append(f"model={self._voice_id}") - params.append(f"encoding={self._settings['encoding']}") + params.append(f"model={self._settings.voice}") + params.append(f"encoding={self._settings.encoding}") params.append(f"sample_rate={self.sample_rate}") url = f"{self._base_url}/v1/speak?{'&'.join(params)}" @@ -330,6 +369,8 @@ class DeepgramHttpTTSService(TTSService): configurable sample rates and quality settings. """ + _settings: DeepgramTTSSettings + def __init__( self, *, @@ -357,10 +398,12 @@ class DeepgramHttpTTSService(TTSService): self._api_key = api_key self._session = aiohttp_session self._base_url = base_url - self._settings = { - "encoding": encoding, - } - self.set_voice(voice) + self._settings = DeepgramTTSSettings( + model=voice, + voice=voice, + encoding=encoding, + ) + self._sync_model_name_to_metrics() def can_generate_metrics(self) -> bool: """Check if the service can generate metrics. @@ -389,8 +432,8 @@ class DeepgramHttpTTSService(TTSService): headers = {"Authorization": f"Token {self._api_key}", "Content-Type": "application/json"} params = { - "model": self._voice_id, - "encoding": self._settings["encoding"], + "model": self._settings.voice, + "encoding": self._settings.encoding, "sample_rate": self.sample_rate, "container": "none", } diff --git a/src/pipecat/services/deepgram/tts_sagemaker.py b/src/pipecat/services/deepgram/tts_sagemaker.py index 7c04bc299..8447c96f0 100644 --- a/src/pipecat/services/deepgram/tts_sagemaker.py +++ b/src/pipecat/services/deepgram/tts_sagemaker.py @@ -14,7 +14,8 @@ streaming audio output. import asyncio import json -from typing import AsyncGenerator, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, Optional from loguru import logger @@ -32,10 +33,22 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.aws.sagemaker.bidi_client import SageMakerBidiClient +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven from pipecat.services.tts_service import TTSService from pipecat.utils.tracing.service_decorators import traced_tts +@dataclass +class DeepgramSageMakerTTSSettings(TTSSettings): + """Settings for Deepgram SageMaker TTS service. + + Parameters: + encoding: Audio encoding format (e.g. "linear16"). + """ + + encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class DeepgramSageMakerTTSService(TTSService): """Deepgram text-to-speech service for AWS SageMaker. @@ -58,6 +71,8 @@ class DeepgramSageMakerTTSService(TTSService): ) """ + _settings: DeepgramSageMakerTTSSettings + def __init__( self, *, @@ -89,8 +104,12 @@ class DeepgramSageMakerTTSService(TTSService): self._endpoint_name = endpoint_name self._region = region - self._encoding = encoding - self.set_voice(voice) + self._settings = DeepgramSageMakerTTSSettings( + model=voice, + voice=voice, + encoding=encoding, + ) + self._sync_model_name_to_metrics() self._client: Optional[SageMakerBidiClient] = None self._response_task: Optional[asyncio.Task] = None @@ -156,7 +175,8 @@ class DeepgramSageMakerTTSService(TTSService): logger.debug("Connecting to Deepgram TTS on SageMaker...") query_string = ( - f"model={self._voice_id}&encoding={self._encoding}&sample_rate={self.sample_rate}" + f"model={self._settings.voice}&encoding={self._settings.encoding}" + f"&sample_rate={self.sample_rate}" ) self._client = SageMakerBidiClient( @@ -200,6 +220,31 @@ class DeepgramSageMakerTTSService(TTSService): logger.debug("Disconnected from Deepgram TTS on SageMaker") await self._call_event_handler("on_disconnected") + async def _update_settings(self, update: TTSSettings) -> dict[str, Any]: + """Apply a settings update and reconnect if necessary. + + Since all settings are part of the SageMaker session query string, + any setting change requires reconnecting to apply the new values. + """ + changed = await super()._update_settings(update) + + if not changed: + return changed + + # Deepgram uses voice as the model, so keep them in sync for metrics + if "voice" in changed: + self._settings.model = self._settings.voice + self._sync_model_name_to_metrics() + + # TODO: someday we could reconnect here to apply updated settings. + # Code might look something like the below: + # await self._disconnect() + # await self._connect() + + self._warn_unhandled_updated_settings(changed) + + return changed + async def _process_responses(self): """Process streaming responses from Deepgram TTS on SageMaker. diff --git a/src/pipecat/services/deepseek/llm.py b/src/pipecat/services/deepseek/llm.py index 56f1ddd18..70318c9ba 100644 --- a/src/pipecat/services/deepseek/llm.py +++ b/src/pipecat/services/deepseek/llm.py @@ -65,18 +65,18 @@ class DeepSeekLLMService(OpenAILLMService): Dictionary of parameters for the chat completion request. """ params = { - "model": self.model_name, + "model": self._settings.model, "stream": True, "stream_options": {"include_usage": True}, - "frequency_penalty": self._settings["frequency_penalty"], - "presence_penalty": self._settings["presence_penalty"], - "temperature": self._settings["temperature"], - "top_p": self._settings["top_p"], - "max_tokens": self._settings["max_tokens"], + "frequency_penalty": self._settings.frequency_penalty, + "presence_penalty": self._settings.presence_penalty, + "temperature": self._settings.temperature, + "top_p": self._settings.top_p, + "max_tokens": self._settings.max_tokens, } # Messages, tools, tool_choice params.update(params_from_context) - params.update(self._settings["extra"]) + params.update(self._settings.extra) return params diff --git a/src/pipecat/services/elevenlabs/stt.py b/src/pipecat/services/elevenlabs/stt.py index 388f7146b..5ff91f597 100644 --- a/src/pipecat/services/elevenlabs/stt.py +++ b/src/pipecat/services/elevenlabs/stt.py @@ -11,11 +11,13 @@ using segmented audio processing. The service uploads audio files and receives transcription results directly. """ +import asyncio import base64 import io import json +from dataclasses import dataclass, field from enum import Enum -from typing import AsyncGenerator, Optional +from typing import Any, AsyncGenerator, Optional import aiohttp from loguru import logger @@ -33,6 +35,7 @@ from pipecat.frames.frames import ( VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven from pipecat.services.stt_latency import ELEVENLABS_REALTIME_TTFS_P99, ELEVENLABS_TTFS_P99 from pipecat.services.stt_service import SegmentedSTTService, WebsocketSTTService from pipecat.transcriptions.language import Language, resolve_language @@ -167,6 +170,51 @@ def language_to_elevenlabs_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +class CommitStrategy(str, Enum): + """Commit strategies for transcript segmentation.""" + + MANUAL = "manual" + VAD = "vad" + + +@dataclass +class ElevenLabsSTTSettings(STTSettings): + """Settings for the ElevenLabs file-based STT service. + + Parameters: + tag_audio_events: Whether to include audio event tags in transcription. + """ + + tag_audio_events: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + +@dataclass +class ElevenLabsRealtimeSTTSettings(STTSettings): + """Settings for the ElevenLabs Realtime STT service. + + See ``ElevenLabsRealtimeSTTService.InputParams`` for detailed descriptions. + + Parameters: + commit_strategy: How to segment speech - manual (Pipecat VAD) or vad (ElevenLabs VAD). + vad_silence_threshold_secs: Seconds of silence before VAD commits (0.3-3.0). + vad_threshold: VAD sensitivity (0.1-0.9, lower is more sensitive). + min_speech_duration_ms: Minimum speech duration for VAD (50-2000ms). + min_silence_duration_ms: Minimum silence duration for VAD (50-2000ms). + include_timestamps: Whether to include word-level timestamps in transcripts. + enable_logging: Whether to enable logging on ElevenLabs' side. + include_language_detection: Whether to include language detection in transcripts. + """ + + commit_strategy: CommitStrategy | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + vad_silence_threshold_secs: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + vad_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + min_speech_duration_ms: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + min_silence_duration_ms: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + include_timestamps: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + enable_logging: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + include_language_detection: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class ElevenLabsSTTService(SegmentedSTTService): """Speech-to-text service using ElevenLabs' file-based API. @@ -175,6 +223,8 @@ class ElevenLabsSTTService(SegmentedSTTService): The service uploads audio files to ElevenLabs and receives transcription results directly. """ + _settings: ElevenLabsSTTSettings + class InputParams(BaseModel): """Configuration parameters for ElevenLabs STT API. @@ -223,13 +273,15 @@ class ElevenLabsSTTService(SegmentedSTTService): self._base_url = base_url self._session = aiohttp_session self._model_id = model - self._tag_audio_events = params.tag_audio_events - self._settings = { - "language": self.language_to_service_language(params.language) + self._settings = ElevenLabsSTTSettings( + model=model, + language=self.language_to_service_language(params.language) if params.language else "eng", - } + tag_audio_events=params.tag_audio_events, + ) + self._sync_model_name_to_metrics() def can_generate_metrics(self) -> bool: """Check if the service can generate processing metrics. @@ -250,27 +302,24 @@ class ElevenLabsSTTService(SegmentedSTTService): """ return language_to_elevenlabs_language(language) - async def set_language(self, language: Language): - """Set the transcription language. + async def _update_settings(self, update: STTSettings) -> dict[str, Any]: + """Apply a settings update. + + Converts language to ElevenLabs format before applying and keeps + ``_model_id`` in sync with the model setting. Args: - language: The language to use for speech-to-text transcription. + update: A :class:`STTSettings` (or ``ElevenLabsSTTSettings``) delta. + + Returns: + Dict mapping changed field names to their previous values. """ - logger.info(f"Switching STT language to: [{language}]") - self._settings["language"] = self.language_to_service_language(language) + changed = await super()._update_settings(update) - async def set_model(self, model: str): - """Set the STT model. + if "model" in changed: + self._model_id = self._settings.model - Args: - model: The model name to use for transcription. - - Note: - ElevenLabs STT API does not currently support model selection. - This method is provided for interface compatibility. - """ - await super().set_model(model) - logger.info(f"Model setting [{model}] noted, but ElevenLabs STT uses default model") + return changed async def _transcribe_audio(self, audio_data: bytes) -> dict: """Upload audio data to ElevenLabs and get transcription result. @@ -298,8 +347,8 @@ class ElevenLabsSTTService(SegmentedSTTService): # Add required model_id, language_code, and tag_audio_events data.add_field("model_id", self._model_id) - data.add_field("language_code", self._settings["language"]) - data.add_field("tag_audio_events", str(self._tag_audio_events).lower()) + data.add_field("language_code", self._settings.language) + data.add_field("tag_audio_events", str(self._settings.tag_audio_events).lower()) async with self._session.post(url, data=data, headers=headers) as response: if response.status != 200: @@ -385,13 +434,6 @@ def audio_format_from_sample_rate(sample_rate: int) -> str: return "pcm_16000" -class CommitStrategy(str, Enum): - """Commit strategies for transcript segmentation.""" - - MANUAL = "manual" - VAD = "vad" - - class ElevenLabsRealtimeSTTService(WebsocketSTTService): """Speech-to-text service using ElevenLabs' Realtime WebSocket API. @@ -404,6 +446,8 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): commit transcript segments, providing consistency with other STT services. """ + _settings: ElevenLabsRealtimeSTTSettings + class InputParams(BaseModel): """Configuration parameters for ElevenLabs Realtime STT API. @@ -469,11 +513,25 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): self._api_key = api_key self._base_url = base_url self._model_id = model - self._params = params self._audio_format = "" # initialized in start() self._receive_task = None - self._settings = {"language": params.language_code} + self._connected_event = asyncio.Event() + self._connected_event.set() + + self._settings = ElevenLabsRealtimeSTTSettings( + model=model, + language=params.language_code, + commit_strategy=params.commit_strategy, + vad_silence_threshold_secs=params.vad_silence_threshold_secs, + vad_threshold=params.vad_threshold, + min_speech_duration_ms=params.min_speech_duration_ms, + min_silence_duration_ms=params.min_silence_duration_ms, + include_timestamps=params.include_timestamps, + enable_logging=params.enable_logging, + include_language_detection=params.include_language_detection, + ) + self._sync_model_name_to_metrics() def can_generate_metrics(self) -> bool: """Check if the service can generate processing metrics. @@ -483,42 +541,29 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): """ return True - async def set_language(self, language: Language): - """Set the transcription language. + async def _update_settings(self, update: STTSettings) -> dict[str, Any]: + """Apply a settings update and reconnect if anything changed. + + Converts language to ElevenLabs format before applying and keeps + ``_model_id`` in sync. Args: - language: The language to use for speech-to-text transcription. + update: A :class:`STTSettings` (or ``ElevenLabsRealtimeSTTSettings``) delta. - Note: - Changing language requires reconnecting to the WebSocket. + Returns: + Dict mapping changed field names to their previous values. """ - logger.info(f"Switching STT language to: [{language}]") - new_language = ( - language_to_elevenlabs_language(language) - if isinstance(language, Language) - else language - ) - self._params.language_code = new_language - self._settings["language"] = new_language - # Reconnect with new settings - await self._disconnect() - await self._connect() - - async def set_model(self, model: str): - """Set the STT model. - - Args: - model: The model name to use for transcription. - - Note: - Changing model requires reconnecting to the WebSocket. - """ - await super().set_model(model) - logger.info(f"Switching STT model to: [{model}]") - self._model_id = model - # Reconnect with new settings + changed = await super()._update_settings(update) + + if not changed: + return changed + + if "model" in changed: + self._model_id = self._settings.model + await self._disconnect() await self._connect() + return changed async def start(self, frame: StartFrame): """Start the STT service and establish WebSocket connection. @@ -566,7 +611,7 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): await self._start_metrics() elif isinstance(frame, VADUserStoppedSpeakingFrame): # Send commit when user stops speaking (manual commit mode) - if self._params.commit_strategy == CommitStrategy.MANUAL: + if self._settings.commit_strategy == CommitStrategy.MANUAL: if self._websocket and self._websocket.state is State.OPEN: try: commit_message = { @@ -589,6 +634,9 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): Yields: None - transcription results are handled via WebSocket responses. """ + # Wait for any in-flight _connect() to finish before checking state + await self._connected_event.wait() + # Reconnect if connection is closed if not self._websocket or self._websocket.state is State.CLOSED: await self._connect() @@ -613,12 +661,18 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): async def _connect(self): """Establish WebSocket connection to ElevenLabs Realtime STT.""" - await self._connect_websocket() + self._connected_event.clear() + try: + await self._connect_websocket() - await super()._connect() + await super()._connect() - if self._websocket and not self._receive_task: - self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) + if self._websocket and not self._receive_task: + self._receive_task = self.create_task( + self._receive_task_handler(self._report_error) + ) + finally: + self._connected_event.set() async def _disconnect(self): """Close WebSocket connection and cleanup tasks.""" @@ -656,36 +710,40 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): # Build query parameters params = [f"model_id={self._model_id}"] - if self._params.language_code: - params.append(f"language_code={self._params.language_code}") + if self._settings.language: + params.append(f"language_code={self._settings.language}") params.append(f"audio_format={self._audio_format}") - params.append(f"commit_strategy={self._params.commit_strategy.value}") + params.append(f"commit_strategy={self._settings.commit_strategy.value}") # Add optional parameters - if self._params.include_timestamps: - params.append(f"include_timestamps={str(self._params.include_timestamps).lower()}") - - if self._params.enable_logging: - params.append(f"enable_logging={str(self._params.enable_logging).lower()}") - - if self._params.include_language_detection: + if self._settings.include_timestamps: params.append( - f"include_language_detection={str(self._params.include_language_detection).lower()}" + f"include_timestamps={str(self._settings.include_timestamps).lower()}" + ) + + if self._settings.enable_logging: + params.append(f"enable_logging={str(self._settings.enable_logging).lower()}") + + if self._settings.include_language_detection: + params.append( + f"include_language_detection={str(self._settings.include_language_detection).lower()}" ) # Add VAD parameters if using VAD commit strategy and values are specified - if self._params.commit_strategy == CommitStrategy.VAD: - if self._params.vad_silence_threshold_secs is not None: + if self._settings.commit_strategy == CommitStrategy.VAD: + if self._settings.vad_silence_threshold_secs is not None: params.append( - f"vad_silence_threshold_secs={self._params.vad_silence_threshold_secs}" + f"vad_silence_threshold_secs={self._settings.vad_silence_threshold_secs}" + ) + if self._settings.vad_threshold is not None: + params.append(f"vad_threshold={self._settings.vad_threshold}") + if self._settings.min_speech_duration_ms is not None: + params.append(f"min_speech_duration_ms={self._settings.min_speech_duration_ms}") + if self._settings.min_silence_duration_ms is not None: + params.append( + f"min_silence_duration_ms={self._settings.min_silence_duration_ms}" ) - if self._params.vad_threshold is not None: - params.append(f"vad_threshold={self._params.vad_threshold}") - if self._params.min_speech_duration_ms is not None: - params.append(f"min_speech_duration_ms={self._params.min_speech_duration_ms}") - if self._params.min_silence_duration_ms is not None: - params.append(f"min_silence_duration_ms={self._params.min_silence_duration_ms}") ws_url = f"wss://{self._base_url}/v1/speech-to-text/realtime?{'&'.join(params)}" @@ -817,7 +875,7 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): """ # If timestamps are enabled, skip this message and wait for the # committed_transcript_with_timestamps message which contains all the data - if self._params.include_timestamps: + if self._settings.include_timestamps: return text = data.get("text", "").strip() diff --git a/src/pipecat/services/elevenlabs/tts.py b/src/pipecat/services/elevenlabs/tts.py index be57a1a3d..72fa6c11a 100644 --- a/src/pipecat/services/elevenlabs/tts.py +++ b/src/pipecat/services/elevenlabs/tts.py @@ -13,7 +13,19 @@ with support for streaming audio, word timestamps, and voice customization. import asyncio import base64 import json -from typing import Any, AsyncGenerator, Dict, List, Literal, Mapping, Optional, Tuple, Union +from dataclasses import dataclass, field +from typing import ( + Any, + AsyncGenerator, + ClassVar, + Dict, + List, + Literal, + Mapping, + Optional, + Tuple, + Union, +) import aiohttp from loguru import logger @@ -32,6 +44,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven, is_given from pipecat.services.tts_service import ( AudioContextWordTTSService, WordTTSService, @@ -136,12 +149,12 @@ def output_format_from_sample_rate(sample_rate: int) -> str: def build_elevenlabs_voice_settings( - settings: Dict[str, Any], + settings: Union[Dict[str, Any], "TTSSettings"], ) -> Optional[Dict[str, Union[float, bool]]]: """Build voice settings dictionary for ElevenLabs based on provided settings. Args: - settings: Dictionary containing voice settings parameters. + settings: Dictionary or settings containing voice settings parameters. Returns: Dictionary of voice settings or None if no valid settings are provided. @@ -150,8 +163,11 @@ def build_elevenlabs_voice_settings( voice_settings = {} for key in voice_setting_keys: - if key in settings and settings[key] is not None: - voice_settings[key] = settings[key] + val = ( + getattr(settings, key, None) if isinstance(settings, TTSSettings) else settings.get(key) + ) + if val is not None and is_given(val): + voice_settings[key] = val return voice_settings or None @@ -168,6 +184,79 @@ class PronunciationDictionaryLocator(BaseModel): version_id: str +@dataclass +class ElevenLabsTTSSettings(TTSSettings): + """Settings for the ElevenLabs WebSocket TTS service. + + Fields that appear in the WebSocket URL (``voice``, ``model``, + ``language``) require a full reconnect when changed. Fields that + affect the voice character (``stability``, ``similarity_boost``, + ``style``, ``use_speaker_boost``, ``speed``) can be applied by closing + the current audio context so a new one is opened with updated settings. + + Parameters: + stability: Voice stability control (0.0 to 1.0). + similarity_boost: Similarity boost control (0.0 to 1.0). + style: Style control for voice expression (0.0 to 1.0). + use_speaker_boost: Whether to use speaker boost enhancement. + speed: Voice speed control (0.7 to 1.2). + auto_mode: Whether to enable automatic mode optimization. + enable_ssml_parsing: Whether to parse SSML tags in text. + enable_logging: Whether to enable ElevenLabs logging. + apply_text_normalization: Text normalization mode ("auto", "on", "off"). + """ + + stability: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + similarity_boost: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + style: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + use_speaker_boost: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + auto_mode: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + enable_ssml_parsing: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + enable_logging: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + apply_text_normalization: Literal["auto", "on", "off"] | None | _NotGiven = field( + default_factory=lambda: NOT_GIVEN + ) + + #: Fields in the WS URL — changing any of these requires a reconnect. + URL_FIELDS: ClassVar[frozenset[str]] = frozenset({"voice", "model", "language"}) + + #: Fields affecting voice character — changing these requires closing the + #: current audio context so the next one picks up new settings. + VOICE_SETTINGS_FIELDS: ClassVar[frozenset[str]] = frozenset( + {"stability", "similarity_boost", "style", "use_speaker_boost", "speed"} + ) + + _aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice"} + + +@dataclass +class ElevenLabsHttpTTSSettings(TTSSettings): + """Settings for the ElevenLabs HTTP TTS service. + + Parameters: + optimize_streaming_latency: Latency optimization level (0-4). + stability: Voice stability control (0.0 to 1.0). + similarity_boost: Similarity boost control (0.0 to 1.0). + style: Style control for voice expression (0.0 to 1.0). + use_speaker_boost: Whether to use speaker boost enhancement. + speed: Voice speed control (0.25 to 4.0). + apply_text_normalization: Text normalization mode ("auto", "on", "off"). + """ + + optimize_streaming_latency: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + stability: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + similarity_boost: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + style: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + use_speaker_boost: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + apply_text_normalization: Literal["auto", "on", "off"] | None | _NotGiven = field( + default_factory=lambda: NOT_GIVEN + ) + + _aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice"} + + def calculate_word_times( alignment_info: Mapping[str, Any], cumulative_time: float, @@ -236,6 +325,8 @@ class ElevenLabsTTSService(AudioContextWordTTSService): customization options including stability, similarity boost, and speed controls. """ + _settings: ElevenLabsTTSSettings + class InputParams(BaseModel): """Input parameters for ElevenLabs TTS configuration. @@ -316,22 +407,24 @@ class ElevenLabsTTSService(AudioContextWordTTSService): self._api_key = api_key self._url = url - self._settings = { - "language": self.language_to_service_language(params.language) - if params.language - else None, - "stability": params.stability, - "similarity_boost": params.similarity_boost, - "style": params.style, - "use_speaker_boost": params.use_speaker_boost, - "speed": params.speed, - "auto_mode": str(params.auto_mode).lower(), - "enable_ssml_parsing": params.enable_ssml_parsing, - "enable_logging": params.enable_logging, - "apply_text_normalization": params.apply_text_normalization, - } - self.set_model_name(model) - self.set_voice(voice_id) + self._settings = ElevenLabsTTSSettings( + model=model, + voice=voice_id, + language=( + self.language_to_service_language(params.language) if params.language else None + ), + stability=params.stability, + similarity_boost=params.similarity_boost, + style=params.style, + use_speaker_boost=params.use_speaker_boost, + speed=params.speed, + auto_mode=str(params.auto_mode).lower(), + enable_ssml_parsing=params.enable_ssml_parsing, + enable_logging=params.enable_logging, + apply_text_normalization=params.apply_text_normalization, + ) + self._sync_model_name_to_metrics() + self._output_format = "" # initialized in start() self._voice_settings = self._set_voice_settings() self._pronunciation_dictionary_locators = params.pronunciation_dictionary_locators @@ -365,54 +458,57 @@ class ElevenLabsTTSService(AudioContextWordTTSService): return language_to_elevenlabs_language(language) def _set_voice_settings(self): - return build_elevenlabs_voice_settings(self._settings) + ts = self._settings + voice_setting_keys = [ + "stability", + "similarity_boost", + "style", + "use_speaker_boost", + "speed", + ] + voice_settings = {} + for key in voice_setting_keys: + val = getattr(ts, key, None) + if val is not None and is_given(val): + voice_settings[key] = val + return voice_settings or None - async def set_model(self, model: str): - """Set the TTS model and reconnect. + async def _update_settings(self, update: TTSSettings) -> dict[str, Any]: + """Apply a settings update, reconnecting as needed. + + Uses the declarative ``URL_FIELDS`` and ``VOICE_SETTINGS_FIELDS`` + sets on :class:`ElevenLabsTTSSettings` to decide whether to + reconnect the WebSocket or close the current audio context. Args: - model: The model name to use for synthesis. + update: A :class:`TTSSettings` (or ``ElevenLabsTTSSettings``) delta. + + Returns: + Dict mapping changed field names to their previous values. """ - await super().set_model(model) - logger.info(f"Switching TTS model to: [{model}]") - await self._disconnect() - await self._connect() + changed = await super()._update_settings(update) - async def _update_settings(self, settings: Mapping[str, Any]): - """Update service settings and reconnect if voice, model, or language changed.""" - # Track previous values for settings that require reconnection - prev_voice = self._voice_id - prev_model = self.model_name - prev_language = self._settings.get("language") - # Create snapshot of current voice settings to detect changes after update - prev_voice_settings = self._voice_settings.copy() if self._voice_settings else None + if not changed: + return changed - await super()._update_settings(settings) - - # Update voice settings for the next context creation + # Rebuild voice settings for next context self._voice_settings = self._set_voice_settings() - # Check if URL-level settings changed (these require reconnection) - url_changed = ( - prev_voice != self._voice_id - or prev_model != self.model_name - or prev_language != self._settings.get("language") - ) - - # Check if only voice settings changed (speed, stability, etc.) - voice_settings_changed = prev_voice_settings != self._voice_settings + url_changed = bool(changed.keys() & ElevenLabsTTSSettings.URL_FIELDS) + voice_settings_changed = bool(changed.keys() & ElevenLabsTTSSettings.VOICE_SETTINGS_FIELDS) if url_changed: - # These settings are in the WebSocket URL, so we need to reconnect logger.debug( - f"URL-level setting changed (voice/model/language), reconnecting WebSocket" + f"URL-level setting changed ({changed.keys() & ElevenLabsTTSSettings.URL_FIELDS}), " + f"reconnecting WebSocket" ) await self._disconnect() await self._connect() elif voice_settings_changed and self.has_active_audio_context(): - # Voice settings can be updated by closing current context - # so new one gets created with updated voice settings - logger.debug(f"Voice settings changed, closing current context to apply changes") + logger.debug( + f"Voice settings changed ({changed.keys() & ElevenLabsTTSSettings.VOICE_SETTINGS_FIELDS}), " + f"closing current context to apply changes" + ) context_id = self.get_active_audio_context_id() try: if self._websocket: @@ -423,6 +519,14 @@ class ElevenLabsTTSService(AudioContextWordTTSService): await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) self.reset_active_audio_context() + if not url_changed: + # Reconnect applies all settings; only warn about fields not handled + # by voice settings or URL changes. + handled = ElevenLabsTTSSettings.URL_FIELDS | ElevenLabsTTSSettings.VOICE_SETTINGS_FIELDS + self._warn_unhandled_updated_settings(changed.keys() - handled) + + return changed + async def start(self, frame: StartFrame): """Start the ElevenLabs TTS service. @@ -503,22 +607,22 @@ class ElevenLabsTTSService(AudioContextWordTTSService): logger.debug("Connecting to ElevenLabs") - voice_id = self._voice_id - model = self.model_name + voice_id = self._settings.voice + model = self._settings.model output_format = self._output_format - url = f"{self._url}/v1/text-to-speech/{voice_id}/multi-stream-input?model_id={model}&output_format={output_format}&auto_mode={self._settings['auto_mode']}" + url = f"{self._url}/v1/text-to-speech/{voice_id}/multi-stream-input?model_id={model}&output_format={output_format}&auto_mode={self._settings.auto_mode}" - if self._settings["enable_ssml_parsing"]: - url += f"&enable_ssml_parsing={self._settings['enable_ssml_parsing']}" + if self._settings.enable_ssml_parsing: + url += f"&enable_ssml_parsing={self._settings.enable_ssml_parsing}" - if self._settings["enable_logging"]: - url += f"&enable_logging={self._settings['enable_logging']}" + if self._settings.enable_logging: + url += f"&enable_logging={self._settings.enable_logging}" - if self._settings["apply_text_normalization"] is not None: - url += f"&apply_text_normalization={self._settings['apply_text_normalization']}" + if self._settings.apply_text_normalization is not None: + url += f"&apply_text_normalization={self._settings.apply_text_normalization}" # Language can only be used with the ELEVENLABS_MULTILINGUAL_MODELS - language = self._settings["language"] + language = self._settings.language if model in ELEVENLABS_MULTILINGUAL_MODELS and language is not None: url += f"&language_code={language}" logger.debug(f"Using language code: {language}") @@ -742,6 +846,8 @@ class ElevenLabsHttpTTSService(WordTTSService): connection is not required or desired. """ + _settings: ElevenLabsHttpTTSSettings + class InputParams(BaseModel): """Input parameters for ElevenLabs HTTP TTS configuration. @@ -808,20 +914,21 @@ class ElevenLabsHttpTTSService(WordTTSService): self._params = params self._session = aiohttp_session - self._settings = { - "language": self.language_to_service_language(params.language) + self._settings = ElevenLabsHttpTTSSettings( + model=model, + voice=voice_id, + language=self.language_to_service_language(params.language) if params.language else None, - "optimize_streaming_latency": params.optimize_streaming_latency, - "stability": params.stability, - "similarity_boost": params.similarity_boost, - "style": params.style, - "use_speaker_boost": params.use_speaker_boost, - "speed": params.speed, - "apply_text_normalization": params.apply_text_normalization, - } - self.set_model_name(model) - self.set_voice(voice_id) + optimize_streaming_latency=params.optimize_streaming_latency, + stability=params.stability, + similarity_boost=params.similarity_boost, + style=params.style, + use_speaker_boost=params.use_speaker_boost, + speed=params.speed, + apply_text_normalization=params.apply_text_normalization, + ) + self._sync_model_name_to_metrics() self._output_format = "" # initialized in start() self._voice_settings = self._set_voice_settings() self._pronunciation_dictionary_locators = params.pronunciation_dictionary_locators @@ -858,10 +965,19 @@ class ElevenLabsHttpTTSService(WordTTSService): def _set_voice_settings(self): return build_elevenlabs_voice_settings(self._settings) - async def _update_settings(self, settings: Mapping[str, Any]): - await super()._update_settings(settings) - # Update voice settings for the next context creation - self._voice_settings = self._set_voice_settings() + async def _update_settings(self, update: TTSSettings) -> dict[str, Any]: + """Apply a settings update and rebuild voice settings. + + Args: + update: A :class:`TTSSettings` (or ``ElevenLabsHttpTTSSettings``) delta. + + Returns: + Dict mapping changed field names to their previous values. + """ + changed = await super()._update_settings(update) + if changed: + self._voice_settings = self._set_voice_settings() + return changed def _reset_state(self): """Reset internal state variables.""" @@ -979,11 +1095,11 @@ class ElevenLabsHttpTTSService(WordTTSService): logger.debug(f"{self}: Generating TTS [{text}]") # Use the with-timestamps endpoint - url = f"{self._base_url}/v1/text-to-speech/{self._voice_id}/stream/with-timestamps" + url = f"{self._base_url}/v1/text-to-speech/{self._settings.voice}/stream/with-timestamps" payload: Dict[str, Union[str, Dict[str, Union[float, bool]]]] = { "text": text, - "model_id": self._model_name, + "model_id": self._settings.model, } # Include previous text as context if available @@ -998,11 +1114,14 @@ class ElevenLabsHttpTTSService(WordTTSService): locator.model_dump() for locator in self._pronunciation_dictionary_locators ] - if self._settings["apply_text_normalization"] is not None: - payload["apply_text_normalization"] = self._settings["apply_text_normalization"] + if ( + is_given(self._settings.apply_text_normalization) + and self._settings.apply_text_normalization is not None + ): + payload["apply_text_normalization"] = self._settings.apply_text_normalization - language = self._settings["language"] - if self._model_name in ELEVENLABS_MULTILINGUAL_MODELS and language: + language = self._settings.language + if self._settings.model in ELEVENLABS_MULTILINGUAL_MODELS and language: payload["language_code"] = language logger.debug(f"Using language code: {language}") elif language: @@ -1019,8 +1138,11 @@ class ElevenLabsHttpTTSService(WordTTSService): params = { "output_format": self._output_format, } - if self._settings["optimize_streaming_latency"] is not None: - params["optimize_streaming_latency"] = self._settings["optimize_streaming_latency"] + if ( + is_given(self._settings.optimize_streaming_latency) + and self._settings.optimize_streaming_latency is not None + ): + params["optimize_streaming_latency"] = self._settings.optimize_streaming_latency try: await self.start_ttfb_metrics() diff --git a/src/pipecat/services/fal/image.py b/src/pipecat/services/fal/image.py index 412cedfbd..fd9d9a22d 100644 --- a/src/pipecat/services/fal/image.py +++ b/src/pipecat/services/fal/image.py @@ -78,7 +78,8 @@ class FalImageGenService(ImageGenService): **kwargs: Additional arguments passed to parent ImageGenService. """ super().__init__(**kwargs) - self.set_model_name(model) + self._settings.model = model + self._sync_model_name_to_metrics() self._params = params self._aiohttp_session = aiohttp_session if key: @@ -103,7 +104,7 @@ class FalImageGenService(ImageGenService): logger.debug(f"Generating image from prompt: {prompt}") response = await fal_client.run_async( - self.model_name, + self._settings.model, arguments={"prompt": prompt, **self._params.model_dump(exclude_none=True)}, ) diff --git a/src/pipecat/services/fal/stt.py b/src/pipecat/services/fal/stt.py index 4e8a655ec..91b5a25c8 100644 --- a/src/pipecat/services/fal/stt.py +++ b/src/pipecat/services/fal/stt.py @@ -11,12 +11,14 @@ transcription using segmented audio processing. """ import os -from typing import AsyncGenerator, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, Optional from loguru import logger from pydantic import BaseModel from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame +from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven from pipecat.services.stt_latency import FAL_TTFS_P99 from pipecat.services.stt_service import SegmentedSTTService from pipecat.transcriptions.language import Language, resolve_language @@ -146,6 +148,22 @@ def language_to_fal_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +@dataclass +class FalSTTSettings(STTSettings): + """Settings for the Fal Wizper STT service. + + Parameters: + task: Task to perform ('transcribe' or 'translate'). Defaults to + 'transcribe'. + chunk_level: Level of chunking ('segment'). Defaults to 'segment'. + version: Version of Wizper model to use. Defaults to '3'. + """ + + task: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + chunk_level: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + version: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class FalSTTService(SegmentedSTTService): """Speech-to-text service using Fal's Wizper API. @@ -153,6 +171,8 @@ class FalSTTService(SegmentedSTTService): segments. It inherits from SegmentedSTTService to handle audio buffering and speech detection. """ + _settings: FalSTTSettings + class InputParams(BaseModel): """Configuration parameters for Fal's Wizper API. @@ -203,14 +223,14 @@ class FalSTTService(SegmentedSTTService): ) self._fal_client = fal_client.AsyncClient(key=api_key or os.getenv("FAL_KEY")) - self._settings = { - "task": params.task, - "language": self.language_to_service_language(params.language) + self._settings = FalSTTSettings( + language=self.language_to_service_language(params.language) if params.language else "en", - "chunk_level": params.chunk_level, - "version": params.version, - } + task=params.task, + chunk_level=params.chunk_level, + version=params.version, + ) def can_generate_metrics(self) -> bool: """Check if the service can generate processing metrics. @@ -231,24 +251,6 @@ class FalSTTService(SegmentedSTTService): """ return language_to_fal_language(language) - async def set_language(self, language: Language): - """Set the transcription language. - - Args: - language: The language to use for speech-to-text transcription. - """ - logger.info(f"Switching STT language to: [{language}]") - self._settings["language"] = self.language_to_service_language(language) - - async def set_model(self, model: str): - """Set the STT model. - - Args: - model: The model name to use for transcription. - """ - await super().set_model(model) - logger.info(f"Switching STT model to: [{model}]") - @traced_stt async def _handle_transcription( self, transcript: str, is_final: bool, language: Optional[str] = None @@ -276,19 +278,19 @@ class FalSTTService(SegmentedSTTService): data_uri = fal_client.encode(audio, "audio/x-wav") response = await self._fal_client.run( "fal-ai/wizper", - arguments={"audio_url": data_uri, **self._settings}, + arguments={"audio_url": data_uri, **self._settings.given_fields()}, ) if response and "text" in response: text = response["text"].strip() if text: # Only yield non-empty text - await self._handle_transcription(text, True, self._settings["language"]) + await self._handle_transcription(text, True, self._settings.language) logger.debug(f"Transcription: [{text}]") yield TranscriptionFrame( text, self._user_id, time_now_iso8601(), - Language(self._settings["language"]), + Language(self._settings.language), result=response, ) diff --git a/src/pipecat/services/fireworks/llm.py b/src/pipecat/services/fireworks/llm.py index d7bf57908..92deb00b9 100644 --- a/src/pipecat/services/fireworks/llm.py +++ b/src/pipecat/services/fireworks/llm.py @@ -66,17 +66,17 @@ class FireworksLLMService(OpenAILLMService): Dictionary of parameters for the chat completion request. """ params = { - "model": self.model_name, + "model": self._settings.model, "stream": True, - "frequency_penalty": self._settings["frequency_penalty"], - "presence_penalty": self._settings["presence_penalty"], - "temperature": self._settings["temperature"], - "top_p": self._settings["top_p"], - "max_tokens": self._settings["max_tokens"], + "frequency_penalty": self._settings.frequency_penalty, + "presence_penalty": self._settings.presence_penalty, + "temperature": self._settings.temperature, + "top_p": self._settings.top_p, + "max_tokens": self._settings.max_tokens, } # Messages, tools, tool_choice params.update(params_from_context) - params.update(self._settings["extra"]) + params.update(self._settings.extra) return params diff --git a/src/pipecat/services/fish/tts.py b/src/pipecat/services/fish/tts.py index 93a718429..0d84ea7ab 100644 --- a/src/pipecat/services/fish/tts.py +++ b/src/pipecat/services/fish/tts.py @@ -11,7 +11,8 @@ for streaming text-to-speech synthesis with customizable voice parameters. """ import uuid -from typing import AsyncGenerator, Literal, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, ClassVar, Dict, Literal, Mapping, Optional from loguru import logger from pydantic import BaseModel @@ -28,6 +29,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven from pipecat.services.tts_service import InterruptibleTTSService from pipecat.transcriptions.language import Language from pipecat.utils.tracing.service_decorators import traced_tts @@ -45,6 +47,41 @@ except ModuleNotFoundError as e: FishAudioOutputFormat = Literal["opus", "mp3", "pcm", "wav"] +@dataclass +class FishAudioTTSSettings(TTSSettings): + """Settings for Fish Audio TTS service. + + Parameters: + fish_sample_rate: Audio sample rate sent to the API. + latency: Latency mode ("normal" or "balanced"). Defaults to "normal". + format: Audio output format. + normalize: Whether to normalize audio output. Defaults to True. + prosody_speed: Speech speed multiplier (0.5-2.0). Defaults to 1.0. + prosody_volume: Volume adjustment in dB. Defaults to 0. + reference_id: Reference ID of the voice model. + """ + + fish_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + latency: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + normalize: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + prosody_speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + prosody_volume: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + reference_id: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + _aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice", "sample_rate": "fish_sample_rate"} + + @classmethod + def from_mapping(cls, settings: Mapping[str, Any]) -> "FishAudioTTSSettings": + """Construct settings from a plain dict, destructuring legacy nested ``prosody``.""" + flat = dict(settings) + nested = flat.pop("prosody", None) + if isinstance(nested, dict): + flat.setdefault("prosody_speed", nested.get("speed")) + flat.setdefault("prosody_volume", nested.get("volume")) + return super().from_mapping(flat) + + class FishAudioTTSService(InterruptibleTTSService): """Fish Audio text-to-speech service with WebSocket streaming. @@ -53,6 +90,8 @@ class FishAudioTTSService(InterruptibleTTSService): audio generation with interruption handling. """ + _settings: FishAudioTTSSettings + class InputParams(BaseModel): """Input parameters for Fish Audio TTS configuration. @@ -136,19 +175,18 @@ class FishAudioTTSService(InterruptibleTTSService): self._receive_task = None self._request_id = None - self._settings = { - "sample_rate": 0, - "latency": params.latency, - "format": output_format, - "normalize": params.normalize, - "prosody": { - "speed": params.prosody_speed, - "volume": params.prosody_volume, - }, - "reference_id": reference_id, - } - - self.set_model_name(model_id) + self._settings = FishAudioTTSSettings( + model=model_id, + voice=reference_id, + fish_sample_rate=0, + latency=params.latency, + format=output_format, + normalize=params.normalize, + prosody_speed=params.prosody_speed, + prosody_volume=params.prosody_volume, + reference_id=reference_id, + ) + self._sync_model_name_to_metrics() def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -158,16 +196,24 @@ class FishAudioTTSService(InterruptibleTTSService): """ return True - async def set_model(self, model: str): - """Set the TTS model and reconnect. + async def _update_settings(self, update: TTSSettings) -> dict[str, Any]: + """Apply a settings update and reconnect if needed. + + Any change to voice or model triggers a WebSocket reconnect. Args: - model: The model name to use for synthesis. + update: A :class:`TTSSettings` (or ``FishAudioTTSSettings``) delta. + + Returns: + Dict mapping changed field names to their previous values. """ - await super().set_model(model) - logger.info(f"Switching TTS model to: [{model}]") - await self._disconnect() - await self._connect() + changed = await super()._update_settings(update) + + if changed: + await self._disconnect() + await self._connect() + + return changed async def start(self, frame: StartFrame): """Start the Fish Audio TTS service. @@ -176,7 +222,7 @@ class FishAudioTTSService(InterruptibleTTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["sample_rate"] = self.sample_rate + self._settings.fish_sample_rate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): @@ -221,11 +267,22 @@ class FishAudioTTSService(InterruptibleTTSService): logger.debug("Connecting to Fish Audio") headers = {"Authorization": f"Bearer {self._api_key}"} - headers["model"] = self.model_name + headers["model"] = self._settings.model self._websocket = await websocket_connect(self._base_url, additional_headers=headers) # Send initial start message with ormsgpack - start_message = {"event": "start", "request": {"text": "", **self._settings}} + request_settings = { + "sample_rate": self._settings.fish_sample_rate, + "latency": self._settings.latency, + "format": self._settings.format, + "normalize": self._settings.normalize, + "prosody": { + "speed": self._settings.prosody_speed, + "volume": self._settings.prosody_volume, + }, + "reference_id": self._settings.reference_id, + } + start_message = {"event": "start", "request": {"text": "", **request_settings}} await self._websocket.send(ormsgpack.packb(start_message)) logger.debug("Sent start message to Fish Audio") diff --git a/src/pipecat/services/gladia/stt.py b/src/pipecat/services/gladia/stt.py index 475a7213e..8d968e00f 100644 --- a/src/pipecat/services/gladia/stt.py +++ b/src/pipecat/services/gladia/stt.py @@ -14,6 +14,7 @@ import asyncio import base64 import json import warnings +from dataclasses import dataclass, field from typing import Any, AsyncGenerator, Dict, Literal, Optional import aiohttp @@ -32,6 +33,7 @@ from pipecat.frames.frames import ( UserStoppedSpeakingFrame, ) from pipecat.services.gladia.config import GladiaInputParams +from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven from pipecat.services.stt_latency import GLADIA_TTFS_P99 from pipecat.services.stt_service import WebsocketSTTService from pipecat.transcriptions.language import Language, resolve_language @@ -178,6 +180,17 @@ class _InputParamsDescriptor: return GladiaInputParams +@dataclass +class GladiaSTTSettings(STTSettings): + """Settings for Gladia STT service. + + Parameters: + input_params: Gladia ``GladiaInputParams`` for detailed configuration. + """ + + input_params: GladiaInputParams | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class GladiaSTTService(WebsocketSTTService): """Speech-to-Text service using Gladia's API. @@ -191,6 +204,8 @@ class GladiaSTTService(WebsocketSTTService): Use :class:`~pipecat.services.gladia.config.GladiaInputParams` directly instead. """ + _settings: GladiaSTTSettings + # Maintain backward compatibility InputParams = _InputParamsDescriptor() @@ -264,10 +279,9 @@ class GladiaSTTService(WebsocketSTTService): self._api_key = api_key self._region = region self._url = url - self.set_model_name(model) - self._params = params self._receive_task = None - self._settings = {} + self._settings = GladiaSTTSettings(model=model, input_params=params) + self._sync_model_name_to_metrics() # Session management self._session_url = None @@ -307,31 +321,33 @@ class GladiaSTTService(WebsocketSTTService): return language_to_gladia_language(language) def _prepare_settings(self) -> Dict[str, Any]: + params = self._settings.input_params + settings = { - "encoding": self._params.encoding or "wav/pcm", - "bit_depth": self._params.bit_depth or 16, + "encoding": params.encoding or "wav/pcm", + "bit_depth": params.bit_depth or 16, "sample_rate": self.sample_rate, - "channels": self._params.channels or 1, - "model": self._model_name, + "channels": params.channels or 1, + "model": self._settings.model, } # Add custom_metadata if provided - settings["custom_metadata"] = dict(self._params.custom_metadata or {}) + settings["custom_metadata"] = dict(params.custom_metadata or {}) settings["custom_metadata"]["pipecat"] = pipecat_version() # Add endpointing parameters if provided - if self._params.endpointing is not None: - settings["endpointing"] = self._params.endpointing - if self._params.maximum_duration_without_endpointing is not None: + if params.endpointing is not None: + settings["endpointing"] = params.endpointing + if params.maximum_duration_without_endpointing is not None: settings["maximum_duration_without_endpointing"] = ( - self._params.maximum_duration_without_endpointing + params.maximum_duration_without_endpointing ) # Add language configuration (prioritize language_config over deprecated language) - if self._params.language_config: - settings["language_config"] = self._params.language_config.model_dump(exclude_none=True) - elif self._params.language: # Backward compatibility for deprecated parameter - language_code = self.language_to_service_language(self._params.language) + if params.language_config: + settings["language_config"] = params.language_config.model_dump(exclude_none=True) + elif params.language: # Backward compatibility for deprecated parameter + language_code = self.language_to_service_language(params.language) if language_code: settings["language_config"] = { "languages": [language_code], @@ -339,21 +355,18 @@ class GladiaSTTService(WebsocketSTTService): } # Add pre_processing configuration if provided - if self._params.pre_processing: - settings["pre_processing"] = self._params.pre_processing.model_dump(exclude_none=True) + if params.pre_processing: + settings["pre_processing"] = params.pre_processing.model_dump(exclude_none=True) # Add realtime_processing configuration if provided - if self._params.realtime_processing: - settings["realtime_processing"] = self._params.realtime_processing.model_dump( + if params.realtime_processing: + settings["realtime_processing"] = params.realtime_processing.model_dump( exclude_none=True ) # Add messages_config if provided - if self._params.messages_config: - settings["messages_config"] = self._params.messages_config.model_dump(exclude_none=True) - - # Store settings for tracing - self._settings = settings + if params.messages_config: + settings["messages_config"] = params.messages_config.model_dump(exclude_none=True) return settings @@ -366,6 +379,33 @@ class GladiaSTTService(WebsocketSTTService): await super().start(frame) await self._connect() + async def _update_settings(self, update: GladiaSTTSettings) -> dict[str, Any]: + """Apply settings update. + + Settings are stored but not applied to the active session. + + Args: + update: A settings delta. + + Returns: + Dict mapping changed field names to their previous values. + """ + changed = await super()._update_settings(update) + + if not changed: + return changed + + # TODO: someday we could reconnect here to apply updated settings. + # Code might look something like the below: + # self._session_url = None + # self._session_id = None + # await self._disconnect() + # await self._connect() + + self._warn_unhandled_updated_settings(changed) + + return changed + async def stop(self, frame: EndFrame): """Stop the Gladia STT websocket connection. @@ -522,7 +562,7 @@ class GladiaSTTService(WebsocketSTTService): Broadcasts UserStartedSpeakingFrame and optionally triggers interruption when VAD is enabled. """ - if not self._params.enable_vad or self._is_speaking: + if not self._settings.input_params.enable_vad or self._is_speaking: return logger.debug(f"{self} User started speaking") @@ -537,7 +577,7 @@ class GladiaSTTService(WebsocketSTTService): Broadcasts UserStoppedSpeakingFrame when VAD is enabled. """ - if not self._params.enable_vad or not self._is_speaking: + if not self._settings.input_params.enable_vad or not self._is_speaking: return self._is_speaking = False await self.broadcast_frame(UserStoppedSpeakingFrame) diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index e209f3d0a..84b06a86d 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -17,9 +17,9 @@ import io import time import uuid import warnings -from dataclasses import dataclass +from dataclasses import dataclass, field from enum import Enum -from typing import Any, Dict, List, Optional, Union +from typing import Any, ClassVar, Dict, List, Optional, Union from loguru import logger from PIL import Image @@ -47,7 +47,6 @@ from pipecat.frames.frames import ( LLMThoughtEndFrame, LLMThoughtStartFrame, LLMThoughtTextFrame, - LLMUpdateSettingsFrame, StartFrame, TranscriptionFrame, TTSAudioRawFrame, @@ -77,6 +76,7 @@ from pipecat.services.openai.llm import ( OpenAIAssistantContextAggregator, OpenAIUserContextAggregator, ) +from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.string import match_endofsentence from pipecat.utils.time import time_now_iso8601 @@ -602,6 +602,33 @@ class InputParams(BaseModel): extra: Optional[Dict[str, Any]] = Field(default_factory=dict) +@dataclass +class GeminiLiveLLMSettings(LLMSettings): + """Settings for Gemini Live LLM services. + + Parameters: + modalities: Response modalities. + language: Language for generation. + media_resolution: Media resolution setting. + vad: Voice activity detection parameters. + context_window_compression: Context window compression configuration. + thinking: Thinking configuration. + enable_affective_dialog: Whether to enable affective dialog. + proactivity: Proactivity configuration. + """ + + modalities: GeminiModalities | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + language: Language | str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + media_resolution: GeminiMediaResolution | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + vad: GeminiVADParams | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + context_window_compression: ContextWindowCompressionParams | dict | _NotGiven = field( + default_factory=lambda: NOT_GIVEN + ) + thinking: ThinkingConfig | dict | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + enable_affective_dialog: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + proactivity: ProactivityConfig | dict | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class GeminiLiveLLMService(LLMService): """Provides access to Google's Gemini Live API. @@ -610,6 +637,8 @@ class GeminiLiveLLMService(LLMService): responses, and tool usage. """ + _settings: GeminiLiveLLMSettings + # Overriding the default adapter to use the Gemini one. adapter_class = GeminiLLMAdapter @@ -672,7 +701,6 @@ class GeminiLiveLLMService(LLMService): self._last_sent_time = 0 self._base_url = base_url - self.set_model_name(model) self._voice_id = voice_id self._language_code = params.language @@ -714,25 +742,27 @@ class GeminiLiveLLMService(LLMService): self._consecutive_failures = 0 self._connection_start_time = None - self._settings = { - "frequency_penalty": params.frequency_penalty, - "max_tokens": params.max_tokens, - "presence_penalty": params.presence_penalty, - "temperature": params.temperature, - "top_k": params.top_k, - "top_p": params.top_p, - "modalities": params.modalities, - "language": self._language_code, - "media_resolution": params.media_resolution, - "vad": params.vad, - "context_window_compression": params.context_window_compression.model_dump() + self._settings = GeminiLiveLLMSettings( + model=model, + frequency_penalty=params.frequency_penalty, + max_tokens=params.max_tokens, + presence_penalty=params.presence_penalty, + temperature=params.temperature, + top_k=params.top_k, + top_p=params.top_p, + modalities=params.modalities, + language=self._language_code, + media_resolution=params.media_resolution, + vad=params.vad, + context_window_compression=params.context_window_compression.model_dump() if params.context_window_compression else {}, - "thinking": params.thinking or {}, - "enable_affective_dialog": params.enable_affective_dialog or False, - "proactivity": params.proactivity or {}, - "extra": params.extra if isinstance(params.extra, dict) else {}, - } + thinking=params.thinking or {}, + enable_affective_dialog=params.enable_affective_dialog or False, + proactivity=params.proactivity or {}, + extra=params.extra if isinstance(params.extra, dict) else {}, + ) + self._sync_model_name_to_metrics() self._file_api_base_url = file_api_base_url self._file_api: Optional[GeminiFileAPI] = None @@ -776,6 +806,25 @@ class GeminiLiveLLMService(LLMService): """ return True + async def _update_settings(self, update: LLMSettings) -> dict[str, Any]: + """Apply a settings update. + + Settings are stored but not applied to the active connection. + """ + changed = await super()._update_settings(update) + + if not changed: + return changed + + # TODO: someday we could reconnect here to apply updated settings. + # Code might look something like the below: + # await self._disconnect() + # await self._connect() + + self._warn_unhandled_updated_settings(changed) + + return changed + def set_audio_input_paused(self, paused: bool): """Set the audio input pause state. @@ -798,7 +847,7 @@ class GeminiLiveLLMService(LLMService): Args: modalities: The modalities to use for responses. """ - self._settings["modalities"] = modalities + self._settings.modalities = modalities def set_language(self, language: Language): """Set the language for generation. @@ -808,7 +857,7 @@ class GeminiLiveLLMService(LLMService): """ self._language = language self._language_code = language_to_gemini_language(language) or "en-US" - self._settings["language"] = self._language_code + self._settings.language = self._language_code logger.info(f"Set Gemini language to: {self._language_code}") async def set_context(self, context: OpenAILLMContext): @@ -866,7 +915,7 @@ class GeminiLiveLLMService(LLMService): async def _handle_interruption(self): if self._bot_is_responding: await self._set_bot_is_responding(False) - if self._settings.get("modalities") == GeminiModalities.AUDIO: + if self._settings.modalities == GeminiModalities.AUDIO: await self.push_frame(TTSStoppedFrame()) # Do not send LLMFullResponseEndFrame here - an interruption # already tells the assistant context aggregator that the response @@ -947,10 +996,9 @@ class GeminiLiveLLMService(LLMService): # uses this frame *without* a user context aggregator still works # (we have an example that does just that, actually). await self._create_single_response(frame.messages) - elif isinstance(frame, LLMUpdateSettingsFrame): - await self._update_settings(frame.settings) elif isinstance(frame, LLMSetToolsFrame): - await self._update_settings() + # TODO: implement runtime tool updates for Gemini Live. + pass else: await self.push_frame(frame, direction) @@ -1074,20 +1122,20 @@ class GeminiLiveLLMService(LLMService): # Assemble basic configuration config = LiveConnectConfig( generation_config=GenerationConfig( - frequency_penalty=self._settings["frequency_penalty"], - max_output_tokens=self._settings["max_tokens"], - presence_penalty=self._settings["presence_penalty"], - temperature=self._settings["temperature"], - top_k=self._settings["top_k"], - top_p=self._settings["top_p"], - response_modalities=[Modality(self._settings["modalities"].value)], + frequency_penalty=self._settings.frequency_penalty, + max_output_tokens=self._settings.max_tokens, + presence_penalty=self._settings.presence_penalty, + temperature=self._settings.temperature, + top_k=self._settings.top_k, + top_p=self._settings.top_p, + response_modalities=[Modality(self._settings.modalities.value)], speech_config=SpeechConfig( voice_config=VoiceConfig( prebuilt_voice_config={"voice_name": self._voice_id} ), - language_code=self._settings["language"], + language_code=self._settings.language, ), - media_resolution=MediaResolution(self._settings["media_resolution"].value), + media_resolution=MediaResolution(self._settings.media_resolution.value), ), input_audio_transcription=AudioTranscriptionConfig(), output_audio_transcription=AudioTranscriptionConfig(), @@ -1095,37 +1143,36 @@ class GeminiLiveLLMService(LLMService): ) # Add context window compression to configuration, if enabled - if self._settings.get("context_window_compression", {}).get("enabled", False): + cwc = self._settings.context_window_compression or {} + if cwc.get("enabled", False): compression_config = ContextWindowCompressionConfig() # Add sliding window (always true if compression is enabled) compression_config.sliding_window = SlidingWindow() # Add trigger_tokens if specified - trigger_tokens = self._settings.get("context_window_compression", {}).get( - "trigger_tokens" - ) + trigger_tokens = cwc.get("trigger_tokens") if trigger_tokens is not None: compression_config.trigger_tokens = trigger_tokens config.context_window_compression = compression_config # Add thinking configuration to configuration, if provided - if self._settings.get("thinking"): - config.thinking_config = self._settings["thinking"] + if self._settings.thinking: + config.thinking_config = self._settings.thinking # Add affective dialog setting, if provided - if self._settings.get("enable_affective_dialog", False): - config.enable_affective_dialog = self._settings["enable_affective_dialog"] + if self._settings.enable_affective_dialog: + config.enable_affective_dialog = self._settings.enable_affective_dialog # Add proactivity configuration to configuration, if provided - if self._settings.get("proactivity"): - config.proactivity = self._settings["proactivity"] + if self._settings.proactivity: + config.proactivity = self._settings.proactivity # Add VAD configuration to configuration, if provided - if self._settings.get("vad"): + if self._settings.vad: vad_config = AutomaticActivityDetection() - vad_params = self._settings["vad"] + vad_params = self._settings.vad has_vad_settings = False # Only add parameters that are explicitly set @@ -1183,7 +1230,9 @@ class GeminiLiveLLMService(LLMService): await self.push_error(error_msg=f"Initialization error: {e}", exception=e) async def _connection_task_handler(self, config: LiveConnectConfig): - async with self._client.aio.live.connect(model=self._model_name, config=config) as session: + async with self._client.aio.live.connect( + model=self._settings.model, config=config + ) as session: logger.info("Connected to Gemini service") # Mark connection start time @@ -1604,7 +1653,7 @@ class GeminiLiveLLMService(LLMService): text: The transcription text to push result: Optional LiveServerMessage that triggered this transcription """ - await self._handle_user_transcription(text, True, self._settings["language"]) + await self._handle_user_transcription(text, True, self._settings.language) await self.push_frame( TranscriptionFrame( text=text, diff --git a/src/pipecat/services/google/image.py b/src/pipecat/services/google/image.py index fcc8e41d0..f03b1da63 100644 --- a/src/pipecat/services/google/image.py +++ b/src/pipecat/services/google/image.py @@ -79,7 +79,9 @@ class GoogleImageGenService(ImageGenService): http_options = update_google_client_http_options(http_options) self._client = genai.Client(api_key=api_key, http_options=http_options) - self.set_model_name(self._params.model) + + self._settings.model = self._params.model + self._sync_model_name_to_metrics() def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index 563acadb3..2004aa15a 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -15,8 +15,8 @@ import io import json import os import uuid -from dataclasses import dataclass -from typing import Any, AsyncIterator, Dict, List, Literal, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncIterator, ClassVar, Dict, List, Literal, Optional from loguru import logger from PIL import Image @@ -39,7 +39,6 @@ from pipecat.frames.frames import ( LLMThoughtEndFrame, LLMThoughtStartFrame, LLMThoughtTextFrame, - LLMUpdateSettingsFrame, ) from pipecat.metrics.metrics import LLMTokenUsage from pipecat.processors.aggregators.llm_context import LLMContext @@ -59,6 +58,7 @@ from pipecat.services.openai.llm import ( OpenAIAssistantContextAggregator, OpenAIUserContextAggregator, ) +from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven, is_given from pipecat.utils.tracing.service_decorators import traced_llm # Suppress gRPC fork warnings @@ -673,6 +673,62 @@ class GoogleLLMContext(OpenAILLMContext): self._messages = [m for m in self._messages if m.parts] +class GoogleThinkingConfig(BaseModel): + """Configuration for controlling the model's internal "thinking" process used before generating a response. + + Gemini 2.5 and 3 series models have this thinking process. + + Parameters: + thinking_level: Thinking level for Gemini 3 models. + For Gemini 3 Pro, this can be "low" or "high". + For Gemini 3 Flash, this can be "minimal", "low", "medium", or "high". + If not provided, Gemini 3 models default to "high". + Note: Gemini 2.5 series must use thinking_budget instead. + thinking_budget: Token budget for thinking, for Gemini 2.5 series. + -1 for dynamic thinking (model decides), 0 to disable thinking, + or a specific token count (e.g., 128-32768 for 2.5 Pro). + If not provided, most models today default to dynamic thinking. + See https://ai.google.dev/gemini-api/docs/thinking#set-budget + for default values and allowed ranges. + Note: Gemini 3 models must use thinking_level instead. + include_thoughts: Whether to include thought summaries in the response. + Today's models default to not including thoughts (False). + """ + + thinking_budget: Optional[int] = Field(default=None) + + # Why `| str` here? To not break compatibility in case Google adds more + # levels in the future. + thinking_level: Optional[Literal["low", "high", "medium", "minimal"] | str] = Field( + default=None + ) + + include_thoughts: Optional[bool] = Field(default=None) + + +@dataclass +class GoogleLLMSettings(LLMSettings): + """Settings for Google LLM services. + + Parameters: + thinking: Thinking configuration. + """ + + thinking: GoogleThinkingConfig | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + @classmethod + def from_mapping(cls, settings): + """Convert a plain dict to settings, coercing thinking dicts. + + For backward compatibility, a ``thinking`` value that is a plain dict + is converted to a :class:`GoogleThinkingConfig`. + """ + instance = super().from_mapping(settings) + if is_given(instance.thinking) and isinstance(instance.thinking, dict): + instance.thinking = GoogleThinkingConfig(**instance.thinking) + return instance + + class GoogleLLMService(LLMService): """Google AI (Gemini) LLM service implementation. @@ -681,40 +737,13 @@ class GoogleLLMService(LLMService): expected by the Google AI model. """ + _settings: GoogleLLMSettings + # Overriding the default adapter to use the Gemini one. adapter_class = GeminiLLMAdapter - class ThinkingConfig(BaseModel): - """Configuration for controlling the model's internal "thinking" process used before generating a response. - - Gemini 2.5 and 3 series models have this thinking process. - - Parameters: - thinking_level: Thinking level for Gemini 3 models. - For Gemini 3 Pro, this can be "low" or "high". - For Gemini 3 Flash, this can be "minimal", "low", "medium", or "high". - If not provided, Gemini 3 models default to "high". - Note: Gemini 2.5 series must use thinking_budget instead. - thinking_budget: Token budget for thinking, for Gemini 2.5 series. - -1 for dynamic thinking (model decides), 0 to disable thinking, - or a specific token count (e.g., 128-32768 for 2.5 Pro). - If not provided, most models today default to dynamic thinking. - See https://ai.google.dev/gemini-api/docs/thinking#set-budget - for default values and allowed ranges. - Note: Gemini 3 models must use thinking_level instead. - include_thoughts: Whether to include thought summaries in the response. - Today's models default to not including thoughts (False). - """ - - thinking_budget: Optional[int] = Field(default=None) - - # Why `| str` here? To not break compatibility in case Google adds more - # levels in the future. - thinking_level: Optional[Literal["low", "high", "medium", "minimal"] | str] = Field( - default=None - ) - - include_thoughts: Optional[bool] = Field(default=None) + # Backward compatibility: ThinkingConfig used to be defined inline here. + ThinkingConfig = GoogleThinkingConfig class InputParams(BaseModel): """Input parameters for Google AI models. @@ -737,7 +766,7 @@ class GoogleLLMService(LLMService): temperature: Optional[float] = Field(default=None, ge=0.0, le=2.0) top_k: Optional[int] = Field(default=None, ge=0) top_p: Optional[float] = Field(default=None, ge=0.0, le=1.0) - thinking: Optional["GoogleLLMService.ThinkingConfig"] = Field(default=None) + thinking: Optional[GoogleThinkingConfig] = Field(default=None) extra: Optional[Dict[str, Any]] = Field(default_factory=dict) def __init__( @@ -768,19 +797,20 @@ class GoogleLLMService(LLMService): params = params or GoogleLLMService.InputParams() - self.set_model_name(model) self._api_key = api_key self._system_instruction = system_instruction self._http_options = update_google_client_http_options(http_options) - self._settings = { - "max_tokens": params.max_tokens, - "temperature": params.temperature, - "top_k": params.top_k, - "top_p": params.top_p, - "thinking": params.thinking, - "extra": params.extra if isinstance(params.extra, dict) else {}, - } + self._settings = GoogleLLMSettings( + model=model, + max_tokens=params.max_tokens, + temperature=params.temperature, + top_k=params.top_k, + top_p=params.top_p, + thinking=params.thinking, + extra=params.extra if isinstance(params.extra, dict) else {}, + ) + self._sync_model_name_to_metrics() self._tools = tools self._tool_config = tool_config @@ -840,7 +870,7 @@ class GoogleLLMService(LLMService): # Use the new google-genai client's async method response = await self._client.aio.models.generate_content( - model=self._model_name, + model=self._settings.model, contents=messages, config=generation_config, ) @@ -874,10 +904,10 @@ class GoogleLLMService(LLMService): k: v for k, v in { "system_instruction": system_instruction, - "temperature": self._settings["temperature"], - "top_p": self._settings["top_p"], - "top_k": self._settings["top_k"], - "max_output_tokens": self._settings["max_tokens"], + "temperature": self._settings.temperature, + "top_p": self._settings.top_p, + "top_k": self._settings.top_k, + "max_output_tokens": self._settings.max_tokens, "tools": tools, "tool_config": tool_config, }.items() @@ -885,13 +915,13 @@ class GoogleLLMService(LLMService): } # Add thinking parameters if configured - if self._settings["thinking"]: - generation_params["thinking_config"] = self._settings["thinking"].model_dump( + if self._settings.thinking: + generation_params["thinking_config"] = self._settings.thinking.model_dump( exclude_unset=True ) - if self._settings["extra"]: - generation_params.update(self._settings["extra"]) + if self._settings.extra: + generation_params.update(self._settings.extra) return generation_params @@ -900,10 +930,10 @@ class GoogleLLMService(LLMService): # There's no way to introspect on model capabilities, so # to check for models that we know default to thinkin on # and can be configured to turn it off. - if not self._model_name.startswith("gemini-2.5-flash"): + if not self._settings.model.startswith("gemini-2.5-flash"): return # If we have an image model, we don't use a budget either. - if "image" in self._model_name: + if "image" in self._settings.model: return # If thinking_config is already set, don't override it. if "thinking_config" in generation_params: @@ -944,7 +974,7 @@ class GoogleLLMService(LLMService): await self.start_ttfb_metrics() return await self._client.aio.models.generate_content_stream( - model=self._model_name, + model=self._settings.model, contents=messages, config=generation_config, ) @@ -1190,8 +1220,6 @@ class GoogleLLMService(LLMService): # NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal # LLMContext with it context = GoogleLLMContext(frame.messages) - elif isinstance(frame, LLMUpdateSettingsFrame): - await self._update_settings(frame.settings) else: await self.push_frame(frame, direction) @@ -1215,14 +1243,6 @@ class GoogleLLMService(LLMService): # Do nothing - we're shutting down anyway pass - async def _update_settings(self, settings): - """Override to handle ThinkingConfig validation.""" - # Convert thinking dict to ThinkingConfig if needed - if "thinking" in settings and isinstance(settings["thinking"], dict): - settings = dict(settings) # Make a copy to avoid modifying the original - settings["thinking"] = self.ThinkingConfig(**settings["thinking"]) - await super()._update_settings(settings) - def create_context_aggregator( self, context: OpenAILLMContext, diff --git a/src/pipecat/services/google/stt.py b/src/pipecat/services/google/stt.py index 23396b0b8..72d4f12b6 100644 --- a/src/pipecat/services/google/stt.py +++ b/src/pipecat/services/google/stt.py @@ -15,13 +15,15 @@ import asyncio import json import os import time +import warnings +from dataclasses import dataclass, field from pipecat.utils.tracing.service_decorators import traced_stt # Suppress gRPC fork warnings os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false" -from typing import AsyncGenerator, List, Optional, Union +from typing import Any, AsyncGenerator, List, Optional, Union from loguru import logger from pydantic import BaseModel, Field, field_validator @@ -34,6 +36,7 @@ from pipecat.frames.frames import ( StartFrame, TranscriptionFrame, ) +from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven from pipecat.services.stt_latency import GOOGLE_TTFS_P99 from pipecat.services.stt_service import STTService from pipecat.transcriptions.language import Language, resolve_language @@ -355,6 +358,46 @@ def language_to_google_stt_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +@dataclass +class GoogleSTTSettings(STTSettings): + """Settings for Google Cloud Speech-to-Text V2. + + Parameters: + languages: List of ``Language`` enums for recognition + (e.g. ``[Language.EN_US]``). Preferred over ``language_codes``. + language_codes: List of Google STT language code strings + (e.g. ``["en-US"]``). + + .. deprecated:: 0.0.103 + Use ``languages`` instead. If both are provided, ``languages`` + takes precedence. This field is here just for backward + compatibility with dict-based settings updates. + use_separate_recognition_per_channel: Process each audio channel separately. + enable_automatic_punctuation: Add punctuation to transcripts. + enable_spoken_punctuation: Include spoken punctuation in transcript. + enable_spoken_emojis: Include spoken emojis in transcript. + profanity_filter: Filter profanity from transcript. + enable_word_time_offsets: Include timing information for each word. + enable_word_confidence: Include confidence scores for each word. + enable_interim_results: Stream partial recognition results. + enable_voice_activity_events: Detect voice activity in audio. + """ + + languages: List[Language] | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + language_codes: List[str] | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + use_separate_recognition_per_channel: bool | _NotGiven = field( + default_factory=lambda: NOT_GIVEN + ) + enable_automatic_punctuation: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + enable_spoken_punctuation: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + enable_spoken_emojis: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + profanity_filter: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + enable_word_time_offsets: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + enable_word_confidence: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + enable_interim_results: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + enable_voice_activity_events: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class GoogleSTTService(STTService): """Google Cloud Speech-to-Text V2 service implementation. @@ -371,6 +414,8 @@ class GoogleSTTService(STTService): ValueError: If project ID is not found in credentials. """ + _settings: GoogleSTTSettings + # Google Cloud's STT service has a connection time limit of 5 minutes per stream. # They've shared an "endless streaming" example that guided this implementation: # https://cloud.google.com/speech-to-text/docs/transcribe-streaming-audio#endless-streaming @@ -508,21 +553,19 @@ class GoogleSTTService(STTService): self._client = speech_v2.SpeechAsyncClient(credentials=creds, client_options=client_options) - self._settings = { - "language_codes": [ - self.language_to_service_language(lang) for lang in params.language_list - ], - "model": params.model, - "use_separate_recognition_per_channel": params.use_separate_recognition_per_channel, - "enable_automatic_punctuation": params.enable_automatic_punctuation, - "enable_spoken_punctuation": params.enable_spoken_punctuation, - "enable_spoken_emojis": params.enable_spoken_emojis, - "profanity_filter": params.profanity_filter, - "enable_word_time_offsets": params.enable_word_time_offsets, - "enable_word_confidence": params.enable_word_confidence, - "enable_interim_results": params.enable_interim_results, - "enable_voice_activity_events": params.enable_voice_activity_events, - } + self._settings = GoogleSTTSettings( + languages=list(params.language_list), + model=params.model, + use_separate_recognition_per_channel=params.use_separate_recognition_per_channel, + enable_automatic_punctuation=params.enable_automatic_punctuation, + enable_spoken_punctuation=params.enable_spoken_punctuation, + enable_spoken_emojis=params.enable_spoken_emojis, + profanity_filter=params.profanity_filter, + enable_word_time_offsets=params.enable_word_time_offsets, + enable_word_confidence=params.enable_word_confidence, + enable_interim_results=params.enable_interim_results, + enable_voice_activity_events=params.enable_voice_activity_events, + ) def can_generate_metrics(self) -> bool: """Check if the service can generate metrics. @@ -545,6 +588,23 @@ class GoogleSTTService(STTService): return [language_to_google_stt_language(lang) or "en-US" for lang in language] return language_to_google_stt_language(language) or "en-US" + def _get_language_codes(self) -> List[str]: + """Resolve the current language settings to Google STT language code strings. + + Prefers ``languages`` (``Language`` enums) over the deprecated + ``language_codes`` (raw strings). Falls back to ``["en-US"]``. + + Returns: + List[str]: Google STT language code strings. + """ + from pipecat.services.settings import is_given + + if is_given(self._settings.languages): + return [self.language_to_service_language(lang) for lang in self._settings.languages] + if is_given(self._settings.language_codes): + return list(self._settings.language_codes) + return ["en-US"] + async def _reconnect_if_needed(self): """Reconnect the stream if it's currently active.""" if self._streaming_task: @@ -552,41 +612,65 @@ class GoogleSTTService(STTService): await self._disconnect() await self._connect() - async def set_language(self, language: Language): - """Update the service's recognition language. - - A convenience method for setting a single language. - - Args: - language: New language for recognition. - """ - logger.debug(f"Switching STT language to: {language}") - await self.set_languages([language]) - async def set_languages(self, languages: List[Language]): """Update the service's recognition languages. + .. deprecated:: + Use ``STTUpdateSettingsFrame`` with ``GoogleSTTSettings(languages=...)`` + instead. + Args: languages: List of languages for recognition. First language is primary. """ + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "set_languages() is deprecated. Use STTUpdateSettingsFrame with " + "GoogleSTTSettings(languages=...) instead.", + DeprecationWarning, + ) logger.debug(f"Switching STT languages to: {languages}") - self._settings["language_codes"] = [ - self.language_to_service_language(lang) for lang in languages - ] - # Recreate stream with new languages - await self._reconnect_if_needed() + await self._update_settings(GoogleSTTSettings(languages=list(languages))) - async def set_model(self, model: str): - """Update the service's recognition model. + async def _update_settings(self, update: GoogleSTTSettings) -> dict[str, Any]: + """Apply settings update and reconnect if anything changed. + + Handles ``language`` from base ``set_language`` by converting it to + ``languages``. Emits a deprecation warning if ``language_codes`` is + used. All other fields (model, boolean flags) are applied directly. + Reconnects the stream on any change. Args: - model: The new recognition model to use. + update: A settings delta. + + Returns: + Dict mapping changed field names to their previous values. """ - logger.debug(f"Switching STT model to: {model}") - await super().set_model(model) - self._settings["model"] = model - # Recreate stream with new model - await self._reconnect_if_needed() + from pipecat.services.settings import is_given + + # If base set_language sent a Language value, convert to languages list + if is_given(update.language): + update.languages = [update.language] + # Clear language so the base class doesn't try to store it + update.language = NOT_GIVEN + + # Warn on deprecated language_codes usage + if is_given(update.language_codes): + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "GoogleSTTSettings.language_codes is deprecated. " + "Use GoogleSTTSettings.languages (List[Language]) instead.", + DeprecationWarning, + stacklevel=2, + ) + + changed = await super()._update_settings(update) + + if changed: + await self._reconnect_if_needed() + + return changed async def start(self, frame: StartFrame): """Start the STT service and establish connection. @@ -632,6 +716,10 @@ class GoogleSTTService(STTService): ) -> None: """Update service options dynamically. + .. deprecated:: + Use ``STTUpdateSettingsFrame`` with ``GoogleSTTSettings(...)`` + instead. + Args: languages: New list of recognition languages. model: New recognition model. @@ -649,55 +737,42 @@ class GoogleSTTService(STTService): Changes that affect the streaming configuration will cause the stream to be reconnected. """ - # Update settings with new values + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "update_options() is deprecated. Use STTUpdateSettingsFrame with " + "GoogleSTTSettings(...) instead.", + DeprecationWarning, + ) + # Build a settings delta from the provided options + update = GoogleSTTSettings() + if languages is not None: - logger.debug(f"Updating language to: {languages}") - self._settings["language_codes"] = [ - self.language_to_service_language(lang) for lang in languages - ] - + update.languages = list(languages) if model is not None: - logger.debug(f"Updating model to: {model}") - self._settings["model"] = model - + update.model = model if enable_automatic_punctuation is not None: - logger.debug(f"Updating automatic punctuation to: {enable_automatic_punctuation}") - self._settings["enable_automatic_punctuation"] = enable_automatic_punctuation - + update.enable_automatic_punctuation = enable_automatic_punctuation if enable_spoken_punctuation is not None: - logger.debug(f"Updating spoken punctuation to: {enable_spoken_punctuation}") - self._settings["enable_spoken_punctuation"] = enable_spoken_punctuation - + update.enable_spoken_punctuation = enable_spoken_punctuation if enable_spoken_emojis is not None: - logger.debug(f"Updating spoken emojis to: {enable_spoken_emojis}") - self._settings["enable_spoken_emojis"] = enable_spoken_emojis - + update.enable_spoken_emojis = enable_spoken_emojis if profanity_filter is not None: - logger.debug(f"Updating profanity filter to: {profanity_filter}") - self._settings["profanity_filter"] = profanity_filter - + update.profanity_filter = profanity_filter if enable_word_time_offsets is not None: - logger.debug(f"Updating word time offsets to: {enable_word_time_offsets}") - self._settings["enable_word_time_offsets"] = enable_word_time_offsets - + update.enable_word_time_offsets = enable_word_time_offsets if enable_word_confidence is not None: - logger.debug(f"Updating word confidence to: {enable_word_confidence}") - self._settings["enable_word_confidence"] = enable_word_confidence - + update.enable_word_confidence = enable_word_confidence if enable_interim_results is not None: - logger.debug(f"Updating interim results to: {enable_interim_results}") - self._settings["enable_interim_results"] = enable_interim_results - + update.enable_interim_results = enable_interim_results if enable_voice_activity_events is not None: - logger.debug(f"Updating voice activity events to: {enable_voice_activity_events}") - self._settings["enable_voice_activity_events"] = enable_voice_activity_events + update.enable_voice_activity_events = enable_voice_activity_events if location is not None: logger.debug(f"Updating location to: {location}") self._location = location - # Reconnect the stream for updates - await self._reconnect_if_needed() + await self._update_settings(update) async def _connect(self): """Initialize streaming recognition config and stream.""" @@ -714,20 +789,20 @@ class GoogleSTTService(STTService): sample_rate_hertz=self.sample_rate, audio_channel_count=1, ), - language_codes=self._settings["language_codes"], - model=self._settings["model"], + language_codes=self._get_language_codes(), + model=self._settings.model, features=cloud_speech.RecognitionFeatures( - enable_automatic_punctuation=self._settings["enable_automatic_punctuation"], - enable_spoken_punctuation=self._settings["enable_spoken_punctuation"], - enable_spoken_emojis=self._settings["enable_spoken_emojis"], - profanity_filter=self._settings["profanity_filter"], - enable_word_time_offsets=self._settings["enable_word_time_offsets"], - enable_word_confidence=self._settings["enable_word_confidence"], + enable_automatic_punctuation=self._settings.enable_automatic_punctuation, + enable_spoken_punctuation=self._settings.enable_spoken_punctuation, + enable_spoken_emojis=self._settings.enable_spoken_emojis, + profanity_filter=self._settings.profanity_filter, + enable_word_time_offsets=self._settings.enable_word_time_offsets, + enable_word_confidence=self._settings.enable_word_confidence, ), ), streaming_features=cloud_speech.StreamingRecognitionFeatures( - enable_voice_activity_events=self._settings["enable_voice_activity_events"], - interim_results=self._settings["enable_interim_results"], + enable_voice_activity_events=self._settings.enable_voice_activity_events, + interim_results=self._settings.enable_interim_results, ), ) @@ -857,7 +932,7 @@ class GoogleSTTService(STTService): if not transcript: continue - primary_language = self._settings["language_codes"][0] + primary_language = self._get_language_codes()[0] if result.is_final: self._last_transcript_was_final = True diff --git a/src/pipecat/services/google/tts.py b/src/pipecat/services/google/tts.py index 4016286df..1103c69e4 100644 --- a/src/pipecat/services/google/tts.py +++ b/src/pipecat/services/google/tts.py @@ -23,7 +23,8 @@ from pipecat.utils.tracing.service_decorators import traced_tts # Suppress gRPC fork warnings os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false" -from typing import Any, AsyncGenerator, List, Literal, Mapping, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, Dict, List, Literal, Optional from loguru import logger from pydantic import BaseModel @@ -36,6 +37,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven, is_given from pipecat.services.tts_service import TTSService from pipecat.transcriptions.language import Language, resolve_language @@ -474,6 +476,71 @@ def language_to_gemini_tts_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +@dataclass +class GoogleHttpTTSSettings(TTSSettings): + """Settings for Google HTTP TTS service. + + Parameters: + pitch: Voice pitch adjustment (e.g., "+2st", "-50%"). + rate: Speaking rate adjustment (e.g., "slow", "fast", "125%"). Used for + SSML prosody tags (non-Chirp voices). + speaking_rate: Speaking rate for AudioConfig (Chirp/Journey voices). + Range [0.25, 2.0]. + volume: Volume adjustment (e.g., "loud", "soft", "+6dB"). + emphasis: Emphasis level for the text. + language: Language for synthesis. Defaults to English. + gender: Voice gender preference. + google_style: Google-specific voice style. + """ + + pitch: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + rate: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + speaking_rate: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + volume: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + emphasis: Literal["strong", "moderate", "reduced", "none"] | None | _NotGiven = field( + default_factory=lambda: NOT_GIVEN + ) + language: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + gender: Literal["male", "female", "neutral"] | None | _NotGiven = field( + default_factory=lambda: NOT_GIVEN + ) + google_style: ( + Literal["apologetic", "calm", "empathetic", "firm", "lively"] | None | _NotGiven + ) = field(default_factory=lambda: NOT_GIVEN) + + +@dataclass +class GoogleStreamTTSSettings(TTSSettings): + """Settings for Google streaming TTS service. + + Parameters: + language: Language for synthesis. Defaults to English. + speaking_rate: The speaking rate, in the range [0.25, 2.0]. + """ + + language: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + speaking_rate: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + +@dataclass +class GeminiTTSSettings(TTSSettings): + """Settings for Gemini TTS service. + + Parameters: + language: Language for synthesis. Defaults to English. + prompt: Optional style instructions for how to synthesize the content. + multi_speaker: Whether to enable multi-speaker support. + speaker_configs: List of speaker configurations for multi-speaker mode. + """ + + language: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + prompt: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + multi_speaker: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + speaker_configs: list[dict[str, Any]] | None | _NotGiven = field( + default_factory=lambda: NOT_GIVEN + ) + + class GoogleHttpTTSService(TTSService): """Google Cloud Text-to-Speech HTTP service with SSML support. @@ -488,6 +555,8 @@ class GoogleHttpTTSService(TTSService): Chirp and Journey voices don't support SSML and will use plain text input. """ + _settings: GoogleHttpTTSSettings + class InputParams(BaseModel): """Input parameters for Google HTTP TTS voice customization. @@ -538,19 +607,19 @@ class GoogleHttpTTSService(TTSService): params = params or GoogleHttpTTSService.InputParams() self._location = location - self._settings = { - "pitch": params.pitch, - "rate": params.rate, - "speaking_rate": params.speaking_rate, - "volume": params.volume, - "emphasis": params.emphasis, - "language": self.language_to_service_language(params.language) + self._settings = GoogleHttpTTSSettings( + pitch=params.pitch, + rate=params.rate, + speaking_rate=params.speaking_rate, + volume=params.volume, + emphasis=params.emphasis, + language=self.language_to_service_language(params.language) if params.language else "en-US", - "gender": params.gender, - "google_style": params.google_style, - } - self.set_voice(voice_id) + gender=params.gender, + google_style=params.google_style, + voice=voice_id, + ) self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client( credentials, credentials_path ) @@ -619,61 +688,60 @@ class GoogleHttpTTSService(TTSService): """ return language_to_google_tts_language(language) - async def _update_settings(self, settings: Mapping[str, Any]): - """Override to handle speaking_rate updates for Chirp/Journey voices. + async def _update_settings(self, update: TTSSettings) -> dict[str, Any]: + """Override to handle speaking_rate validation. Args: - settings: Dictionary of settings to update. Can include 'speaking_rate' (float) + update: Settings delta. Can include 'speaking_rate' (float). """ - if "speaking_rate" in settings: - rate_value = float(settings["speaking_rate"]) - if 0.25 <= rate_value <= 2.0: - self._settings["speaking_rate"] = rate_value - else: + if isinstance(update, GoogleHttpTTSSettings) and is_given(update.speaking_rate): + rate_value = float(update.speaking_rate) + if not (0.25 <= rate_value <= 2.0): logger.warning( f"Invalid speaking_rate value: {rate_value}. Must be between 0.25 and 2.0" ) - await super()._update_settings(settings) + update.speaking_rate = NOT_GIVEN + return await super()._update_settings(update) def _construct_ssml(self, text: str) -> str: ssml = "" # Voice tag - voice_attrs = [f"name='{self._voice_id}'"] + voice_attrs = [f"name='{self._settings.voice}'"] - language = self._settings["language"] + language = self._settings.language voice_attrs.append(f"language='{language}'") - if self._settings["gender"]: - voice_attrs.append(f"gender='{self._settings['gender']}'") + if self._settings.gender: + voice_attrs.append(f"gender='{self._settings.gender}'") ssml += f"" # Prosody tag prosody_attrs = [] - if self._settings["pitch"]: - prosody_attrs.append(f"pitch='{self._settings['pitch']}'") - if self._settings["rate"]: - prosody_attrs.append(f"rate='{self._settings['rate']}'") - if self._settings["volume"]: - prosody_attrs.append(f"volume='{self._settings['volume']}'") + if self._settings.pitch: + prosody_attrs.append(f"pitch='{self._settings.pitch}'") + if self._settings.rate: + prosody_attrs.append(f"rate='{self._settings.rate}'") + if self._settings.volume: + prosody_attrs.append(f"volume='{self._settings.volume}'") if prosody_attrs: ssml += f"" # Emphasis tag - if self._settings["emphasis"]: - ssml += f"" + if self._settings.emphasis: + ssml += f"" # Google style tag - if self._settings["google_style"]: - ssml += f"" + if self._settings.google_style: + ssml += f"" ssml += text # Close tags - if self._settings["google_style"]: + if self._settings.google_style: ssml += "" - if self._settings["emphasis"]: + if self._settings.emphasis: ssml += "" if prosody_attrs: ssml += "" @@ -698,8 +766,8 @@ class GoogleHttpTTSService(TTSService): await self.start_ttfb_metrics() # Check if the voice is a Chirp voice (including Chirp 3) or Journey voice - is_chirp_voice = "chirp" in self._voice_id.lower() - is_journey_voice = "journey" in self._voice_id.lower() + is_chirp_voice = "chirp" in self._settings.voice.lower() + is_journey_voice = "journey" in self._settings.voice.lower() # Create synthesis input based on voice_id if is_chirp_voice or is_journey_voice: @@ -710,7 +778,7 @@ class GoogleHttpTTSService(TTSService): synthesis_input = texttospeech_v1.SynthesisInput(ssml=ssml) voice = texttospeech_v1.VoiceSelectionParams( - language_code=self._settings["language"], name=self._voice_id + language_code=self._settings.language, name=self._settings.voice ) # Build audio config with conditional speaking_rate audio_config_params = { @@ -719,8 +787,8 @@ class GoogleHttpTTSService(TTSService): } # For Chirp and Journey voices, include speaking_rate in AudioConfig - if (is_chirp_voice or is_journey_voice) and self._settings["speaking_rate"] is not None: - audio_config_params["speaking_rate"] = self._settings["speaking_rate"] + if (is_chirp_voice or is_journey_voice) and self._settings.speaking_rate is not None: + audio_config_params["speaking_rate"] = self._settings.speaking_rate audio_config = texttospeech_v1.AudioConfig(**audio_config_params) @@ -910,6 +978,8 @@ class GoogleTTSService(GoogleBaseTTSService): ) """ + _settings: GoogleStreamTTSSettings + class InputParams(BaseModel): """Input parameters for Google streaming TTS configuration. @@ -950,33 +1020,32 @@ class GoogleTTSService(GoogleBaseTTSService): params = params or GoogleTTSService.InputParams() self._location = location - self._settings = { - "language": self.language_to_service_language(params.language) + self._settings = GoogleStreamTTSSettings( + language=self.language_to_service_language(params.language) if params.language else "en-US", - "speaking_rate": params.speaking_rate, - } - self.set_voice(voice_id) + speaking_rate=params.speaking_rate, + voice=voice_id, + ) self._voice_cloning_key = voice_cloning_key self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client( credentials, credentials_path ) - async def _update_settings(self, settings: Mapping[str, Any]): - """Override to handle speaking_rate updates for streaming API. + async def _update_settings(self, update: TTSSettings) -> dict[str, Any]: + """Override to handle speaking_rate validation. Args: - settings: Dictionary of settings to update. Can include 'speaking_rate' (float) + update: Settings delta. Can include 'speaking_rate' (float). """ - if "speaking_rate" in settings: - rate_value = float(settings["speaking_rate"]) - if 0.25 <= rate_value <= 2.0: - self._settings["speaking_rate"] = rate_value - else: + if isinstance(update, GoogleStreamTTSSettings) and is_given(update.speaking_rate): + rate_value = float(update.speaking_rate) + if not (0.25 <= rate_value <= 2.0): logger.warning( f"Invalid speaking_rate value: {rate_value}. Must be between 0.25 and 2.0" ) - await super()._update_settings(settings) + update.speaking_rate = NOT_GIVEN + return await super()._update_settings(update) @traced_tts async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]: @@ -1000,11 +1069,11 @@ class GoogleTTSService(GoogleBaseTTSService): voice_cloning_key=self._voice_cloning_key ) voice = texttospeech_v1.VoiceSelectionParams( - language_code=self._settings["language"], voice_clone=voice_clone_params + language_code=self._settings.language, voice_clone=voice_clone_params ) else: voice = texttospeech_v1.VoiceSelectionParams( - language_code=self._settings["language"], name=self._voice_id + language_code=self._settings.language, name=self._settings.voice ) # Create streaming config @@ -1013,7 +1082,7 @@ class GoogleTTSService(GoogleBaseTTSService): streaming_audio_config=texttospeech_v1.StreamingAudioConfig( audio_encoding=texttospeech_v1.AudioEncoding.PCM, sample_rate_hertz=self.sample_rate, - speaking_rate=self._settings["speaking_rate"], + speaking_rate=self._settings.speaking_rate, ), ) @@ -1052,6 +1121,8 @@ class GeminiTTSService(GoogleBaseTTSService): ) """ + _settings: GeminiTTSSettings + GOOGLE_SAMPLE_RATE = 24000 # Google TTS always outputs at 24kHz # List of available Gemini TTS voices @@ -1158,15 +1229,15 @@ class GeminiTTSService(GoogleBaseTTSService): self._location = location self._model = model - self._voice_id = voice_id - self._settings = { - "language": self.language_to_service_language(params.language) + self._settings = GeminiTTSSettings( + language=self.language_to_service_language(params.language) if params.language else "en-US", - "prompt": params.prompt, - "multi_speaker": params.multi_speaker, - "speaker_configs": params.speaker_configs, - } + prompt=params.prompt, + multi_speaker=params.multi_speaker, + speaker_configs=params.speaker_configs, + voice=voice_id, + ) self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client( credentials, credentials_path @@ -1183,16 +1254,6 @@ class GeminiTTSService(GoogleBaseTTSService): """ return language_to_gemini_tts_language(language) - def set_voice(self, voice_id: str): - """Set the voice for TTS generation. - - Args: - voice_id: Name of the voice to use from AVAILABLE_VOICES. - """ - if voice_id not in self.AVAILABLE_VOICES: - logger.warning(f"Voice '{voice_id}' not in known voices list. Using anyway.") - self._voice_id = voice_id - async def start(self, frame: StartFrame): """Start the Gemini TTS service. @@ -1206,15 +1267,19 @@ class GeminiTTSService(GoogleBaseTTSService): f"Current rate of {self.sample_rate}Hz may cause issues." ) - async def _update_settings(self, settings: Mapping[str, Any]): - """Override to handle prompt updates. + async def _update_settings(self, update: TTSSettings) -> dict[str, Any]: + """Apply a settings update with voice validation. Args: - settings: Dictionary of settings to update. Can include 'prompt' (str) + update: Settings delta. Can include 'voice', 'prompt', etc. + + Returns: + Dict mapping changed field names to their previous values. """ - if "prompt" in settings: - self._settings["prompt"] = settings["prompt"] - await super()._update_settings(settings) + if is_given(update.voice) and update.voice not in self.AVAILABLE_VOICES: + logger.warning(f"Voice '{update.voice}' not in known voices list. Using anyway.") + + return await super()._update_settings(update) @traced_tts async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]: @@ -1234,14 +1299,14 @@ class GeminiTTSService(GoogleBaseTTSService): await self.start_ttfb_metrics() # Build voice selection params - if self._settings["multi_speaker"] and self._settings["speaker_configs"]: + if self._settings.multi_speaker and self._settings.speaker_configs: # Multi-speaker mode speaker_voice_configs = [] - for speaker_config in self._settings["speaker_configs"]: + for speaker_config in self._settings.speaker_configs: speaker_voice_configs.append( texttospeech_v1.MultispeakerPrebuiltVoice( speaker_alias=speaker_config["speaker_alias"], - speaker_id=speaker_config.get("speaker_id", self._voice_id), + speaker_id=speaker_config.get("speaker_id", self._settings.voice), ) ) @@ -1250,15 +1315,15 @@ class GeminiTTSService(GoogleBaseTTSService): ) voice = texttospeech_v1.VoiceSelectionParams( - language_code=self._settings["language"], + language_code=self._settings.language, model_name=self._model, multi_speaker_voice_config=multi_speaker_voice_config, ) else: # Single speaker mode voice = texttospeech_v1.VoiceSelectionParams( - language_code=self._settings["language"], - name=self._voice_id, + language_code=self._settings.language, + name=self._settings.voice, model_name=self._model, ) @@ -1273,7 +1338,7 @@ class GeminiTTSService(GoogleBaseTTSService): # Use base class streaming logic with prompt support async for frame in self._stream_tts( - streaming_config, text, context_id, self._settings["prompt"] + streaming_config, text, context_id, self._settings.prompt ): yield frame diff --git a/src/pipecat/services/gradium/stt.py b/src/pipecat/services/gradium/stt.py index 7433c2549..1583fac3c 100644 --- a/src/pipecat/services/gradium/stt.py +++ b/src/pipecat/services/gradium/stt.py @@ -12,7 +12,8 @@ WebSocket API for streaming audio transcription. import base64 import json -from typing import AsyncGenerator, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, Optional from loguru import logger from pydantic import BaseModel @@ -27,6 +28,7 @@ from pipecat.frames.frames import ( VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven, is_given from pipecat.services.stt_latency import GRADIUM_TTFS_P99 from pipecat.services.stt_service import WebsocketSTTService from pipecat.transcriptions.language import Language, resolve_language @@ -64,6 +66,18 @@ def language_to_gradium_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +@dataclass +class GradiumSTTSettings(STTSettings): + """Settings for the Gradium STT service. + + Parameters: + delay_in_frames: Delay in audio frames (80ms each) before text is + generated. Higher delays allow more context but increase latency. + """ + + delay_in_frames: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class GradiumSTTService(WebsocketSTTService): """Gradium real-time speech-to-text service. @@ -72,6 +86,8 @@ class GradiumSTTService(WebsocketSTTService): for audio processing and connection management. """ + _settings: GradiumSTTSettings + class InputParams(BaseModel): """Configuration parameters for Gradium STT API. @@ -127,9 +143,15 @@ class GradiumSTTService(WebsocketSTTService): self._api_key = api_key self._api_endpoint_base_url = api_endpoint_base_url self._websocket = None - self._params = params or GradiumSTTService.InputParams() self._json_config = json_config + params = params or GradiumSTTService.InputParams() + + self._settings = GradiumSTTSettings( + language=params.language, + delay_in_frames=params.delay_in_frames if params.delay_in_frames else NOT_GIVEN, + ) + self._receive_task = None self._audio_buffer = bytearray() @@ -149,16 +171,22 @@ class GradiumSTTService(WebsocketSTTService): """ return True - async def set_language(self, language: Language): - """Set the recognition language and reconnect. + async def _update_settings(self, update: STTSettings) -> dict[str, Any]: + """Apply a settings update, sync params, and reconnect. Args: - language: The language to use for speech recognition. + update: A :class:`STTSettings` (or ``GradiumSTTSettings``) delta. + + Returns: + Dict mapping changed field names to their previous values. """ - logger.info(f"Switching STT language to: [{language}]") - self._params.language = language + changed = await super()._update_settings(update) + if not changed: + return changed + await self._disconnect() await self._connect() + return changed async def start(self, frame: StartFrame): """Start the speech-to-text service. @@ -298,12 +326,12 @@ class GradiumSTTService(WebsocketSTTService): json_config = {} if self._json_config: json_config = json.loads(self._json_config) - if self._params.language: - gradium_language = language_to_gradium_language(self._params.language) + if is_given(self._settings.language) and self._settings.language: + gradium_language = language_to_gradium_language(self._settings.language) if gradium_language: json_config["language"] = gradium_language - if self._params.delay_in_frames: - json_config["delay_in_frames"] = self._params.delay_in_frames + if is_given(self._settings.delay_in_frames) and self._settings.delay_in_frames: + json_config["delay_in_frames"] = self._settings.delay_in_frames if json_config: setup_msg["json_config"] = json_config await self._websocket.send(json.dumps(setup_msg)) diff --git a/src/pipecat/services/gradium/tts.py b/src/pipecat/services/gradium/tts.py index 98e08a9d3..8b8995c41 100644 --- a/src/pipecat/services/gradium/tts.py +++ b/src/pipecat/services/gradium/tts.py @@ -6,7 +6,8 @@ import base64 import json -from typing import Any, AsyncGenerator, Mapping, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, Optional from loguru import logger from pydantic import BaseModel @@ -23,6 +24,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven from pipecat.services.tts_service import AudioContextWordTTSService from pipecat.utils.tracing.service_decorators import traced_tts @@ -38,9 +40,22 @@ except ModuleNotFoundError as e: SAMPLE_RATE = 48000 +@dataclass +class GradiumTTSSettings(TTSSettings): + """Settings for the Gradium TTS service. + + Parameters: + output_format: Audio output format. + """ + + output_format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class GradiumTTSService(AudioContextWordTTSService): """Text-to-Speech service using Gradium's websocket API.""" + _settings: GradiumTTSSettings + class InputParams(BaseModel): """Configuration parameters for Gradium TTS service. @@ -85,14 +100,12 @@ class GradiumTTSService(AudioContextWordTTSService): # Store service configuration self._api_key = api_key self._url = url - self._voice_id = voice_id self._json_config = json_config - self._model = model - self._settings = { - "voice_id": voice_id, - "model_name": model, - "output_format": "pcm", - } + self._settings = GradiumTTSSettings( + model=model, + voice=voice_id, + output_format="pcm", + ) # State tracking self._receive_task = None @@ -105,24 +118,22 @@ class GradiumTTSService(AudioContextWordTTSService): """ return True - async def set_model(self, model: str): - """Update the TTS model. + async def _update_settings(self, update: TTSSettings) -> dict[str, Any]: + """Apply a settings update and reconnect if voice changed. Args: - model: The model name to use for synthesis. - """ - self._model = model - await super().set_model(model) + update: A :class:`TTSSettings` (or ``GradiumTTSSettings``) delta. - async def _update_settings(self, settings: Mapping[str, Any]): - """Update service settings and reconnect if voice changed.""" - prev_voice = self._voice_id - await super()._update_settings(settings) - if not prev_voice == self._voice_id: - self._settings["voice_id"] = self._voice_id - logger.info(f"Switching TTS voice to: [{self._voice_id}]") + Returns: + Dict mapping changed field names to their previous values. + """ + changed = await super()._update_settings(update) + if "voice" in changed: await self._disconnect() await self._connect() + else: + self._warn_unhandled_updated_settings(changed) + return changed def _build_msg(self, text: str = "") -> dict: """Build JSON message for Gradium API.""" @@ -200,7 +211,7 @@ class GradiumTTSService(AudioContextWordTTSService): setup_msg = { "type": "setup", "output_format": "pcm", - "voice_id": self._voice_id, + "voice_id": self._settings.voice, "close_ws_on_eos": False, } if self._json_config is not None: diff --git a/src/pipecat/services/grok/realtime/llm.py b/src/pipecat/services/grok/realtime/llm.py index e1355ce31..14c93c94a 100644 --- a/src/pipecat/services/grok/realtime/llm.py +++ b/src/pipecat/services/grok/realtime/llm.py @@ -13,8 +13,8 @@ https://docs.x.ai/docs/guides/voice/agent import base64 import json import time -from dataclasses import dataclass -from typing import Optional +from dataclasses import dataclass, field +from typing import Any, Optional from loguru import logger @@ -56,6 +56,7 @@ from pipecat.processors.aggregators.llm_response_universal import ( from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import FunctionCallFromLLM, LLMService +from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven from pipecat.utils.time import time_now_iso8601 from . import events @@ -85,6 +86,19 @@ class CurrentAudioResponse: total_size: int = 0 +@dataclass +class GrokRealtimeLLMSettings(LLMSettings): + """Settings for Grok Realtime LLM services. + + Parameters: + session_properties: Grok Realtime session configuration. + """ + + session_properties: events.SessionProperties | _NotGiven = field( + default_factory=lambda: NOT_GIVEN + ) + + class GrokRealtimeLLMService(LLMService): """Grok Realtime Voice Agent LLM service providing real-time audio and text communication. @@ -101,6 +115,8 @@ class GrokRealtimeLLMService(LLMService): - Server-side VAD (Voice Activity Detection) """ + _settings: GrokRealtimeLLMSettings + # Use the Grok-specific adapter adapter_class = GrokRealtimeLLMAdapter @@ -134,9 +150,8 @@ class GrokRealtimeLLMService(LLMService): self.api_key = api_key self.base_url = base_url - # Initialize session_properties - self._session_properties: events.SessionProperties = ( - session_properties or events.SessionProperties() + self._settings = GrokRealtimeLLMSettings( + session_properties=session_properties or events.SessionProperties(), ) self._audio_input_paused = start_audio_paused @@ -186,13 +201,13 @@ class GrokRealtimeLLMService(LLMService): Configured sample rate or None if not manually configured. For PCMU/PCMA formats, returns 8000 Hz (G.711 standard). """ - if not self._session_properties.audio: + if not self._settings.session_properties.audio: return None audio_config = ( - self._session_properties.audio.input + self._settings.session_properties.audio.input if direction == "input" - else self._session_properties.audio.output + else self._settings.session_properties.audio.output ) if audio_config and audio_config.format: @@ -222,8 +237,8 @@ class GrokRealtimeLLMService(LLMService): def _is_turn_detection_enabled(self) -> bool: """Check if server-side VAD is enabled.""" - if self._session_properties.turn_detection: - return self._session_properties.turn_detection.type == "server_vad" + if self._settings.session_properties.turn_detection: + return self._settings.session_properties.turn_detection.type == "server_vad" return False async def _handle_interruption(self): @@ -281,6 +296,27 @@ class GrokRealtimeLLMService(LLMService): # Standard AIService frame handling # + def _ensure_audio_config(self, input_sample_rate: int, output_sample_rate: int): + """Ensure session_properties.audio has input and output configs. + + Fills in any missing audio configuration using the given sample rates. + + Args: + input_sample_rate: Sample rate for audio input (Hz). + output_sample_rate: Sample rate for audio output (Hz). + """ + props = self._settings.session_properties + if not props.audio: + props.audio = events.AudioConfiguration() + if not props.audio.input: + props.audio.input = events.AudioInput( + format=events.PCMAudioFormat(rate=input_sample_rate) + ) + if not props.audio.output: + props.audio.output = events.AudioOutput( + format=events.PCMAudioFormat(rate=output_sample_rate) + ) + async def start(self, frame: StartFrame): """Start the service and establish WebSocket connection. @@ -288,23 +324,7 @@ class GrokRealtimeLLMService(LLMService): frame: The start frame triggering service initialization. """ await super().start(frame) - - # Ensure audio configuration exists with both input and output - if not self._session_properties.audio: - self._session_properties.audio = events.AudioConfiguration() - - # Fill in missing input configuration - if not self._session_properties.audio.input: - self._session_properties.audio.input = events.AudioInput( - format=events.PCMAudioFormat(rate=frame.audio_in_sample_rate) - ) - - # Fill in missing output configuration - if not self._session_properties.audio.output: - self._session_properties.audio.output = events.AudioOutput( - format=events.PCMAudioFormat(rate=frame.audio_out_sample_rate) - ) - + self._ensure_audio_config(frame.audio_in_sample_rate, frame.audio_out_sample_rate) await self._connect() async def stop(self, frame: EndFrame): @@ -336,6 +356,16 @@ class GrokRealtimeLLMService(LLMService): frame: The frame to process. direction: The direction of frame flow in the pipeline. """ + # Backward-compatible dict path: frame.settings contains SessionProperties + # fields, not our Settings fields, so we construct SessionProperties + # directly. The frame.update path falls through to super, which calls + # _update_settings → our override handles the rest. + if isinstance(frame, LLMUpdateSettingsFrame) and frame.update is None: + self._settings.session_properties = events.SessionProperties(**frame.settings) + await self._send_session_update() + await self.push_frame(frame, direction) + return + await super().process_frame(frame, direction) if isinstance(frame, TranscriptionFrame): @@ -355,11 +385,8 @@ class GrokRealtimeLLMService(LLMService): await self._handle_bot_stopped_speaking() elif isinstance(frame, LLMMessagesAppendFrame): await self._handle_messages_append(frame) - elif isinstance(frame, LLMUpdateSettingsFrame): - self._session_properties = events.SessionProperties(**frame.settings) - await self._update_settings() elif isinstance(frame, LLMSetToolsFrame): - await self._update_settings() + await self._send_session_update() await self.push_frame(frame, direction) @@ -436,9 +463,30 @@ class GrokRealtimeLLMService(LLMService): return await self.push_error(error_msg=f"Error sending client event: {e}", exception=e) - async def _update_settings(self): + async def _update_settings(self, update): + """Apply a settings update, sending a session update if needed.""" + # Capture current sample rates before the update replaces them. + input_rate = self._get_configured_sample_rate("input") + output_rate = self._get_configured_sample_rate("output") + + changed = await super()._update_settings(update) + + if "session_properties" in changed: + if input_rate and output_rate: + self._ensure_audio_config(input_rate, output_rate) + else: + logger.warning( + "Attempting to apply session properties update without configured sample rates. " + "Audio configuration may be incomplete." + ) + await self._send_session_update() + + self._warn_unhandled_updated_settings(changed.keys() - {"session_properties"}) + return changed + + async def _send_session_update(self): """Update session settings on the server.""" - settings = self._session_properties + settings = self._settings.session_properties adapter: GrokRealtimeLLMAdapter = self.get_llm_adapter() if self._context: @@ -516,7 +564,7 @@ class GrokRealtimeLLMService(LLMService): async def _handle_evt_conversation_created(self, evt): """Handle conversation.created event - first event after connecting.""" - await self._update_settings() + await self._send_session_update() async def _handle_evt_response_created(self, evt): """Handle response.created event - response generation started.""" @@ -719,7 +767,7 @@ class GrokRealtimeLLMService(LLMService): self._messages_added_manually[evt.item.id] = True await self.send_client_event(evt) - await self._update_settings() + await self._send_session_update() self._llm_needs_conversation_setup = False logger.debug("Creating Grok response") diff --git a/src/pipecat/services/groq/stt.py b/src/pipecat/services/groq/stt.py index 52cb0a7cc..d51e93c68 100644 --- a/src/pipecat/services/groq/stt.py +++ b/src/pipecat/services/groq/stt.py @@ -62,7 +62,7 @@ class GroqSTTService(BaseWhisperSTTService): # Build kwargs dict with only set parameters kwargs = { "file": ("audio.wav", audio, "audio/wav"), - "model": self.model_name, + "model": self._settings.model, # Use verbose_json to get probability metrics "response_format": "verbose_json" if self._include_prob_metrics else "json", "language": self._language, diff --git a/src/pipecat/services/groq/tts.py b/src/pipecat/services/groq/tts.py index 331af8eb7..cc073f8c7 100644 --- a/src/pipecat/services/groq/tts.py +++ b/src/pipecat/services/groq/tts.py @@ -8,7 +8,8 @@ import io import wave -from typing import AsyncGenerator, Optional +from dataclasses import dataclass, field +from typing import AsyncGenerator, ClassVar, Dict, Optional from loguru import logger from pydantic import BaseModel @@ -20,6 +21,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven from pipecat.services.tts_service import TTSService from pipecat.transcriptions.language import Language from pipecat.utils.tracing.service_decorators import traced_tts @@ -32,6 +34,23 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class GroqTTSSettings(TTSSettings): + """Settings for the Groq TTS service. + + Parameters: + output_format: Audio output format. + speed: Speech speed multiplier. Defaults to 1.0. + groq_sample_rate: Audio sample rate. + """ + + output_format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + groq_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + _aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice", "sample_rate": "groq_sample_rate"} + + class GroqTTSService(TTSService): """Groq text-to-speech service implementation. @@ -40,6 +59,8 @@ class GroqTTSService(TTSService): and output formats. """ + _settings: GroqTTSSettings + class InputParams(BaseModel): """Input parameters for Groq TTS configuration. @@ -87,19 +108,18 @@ class GroqTTSService(TTSService): params = params or GroqTTSService.InputParams() self._api_key = api_key - self._model_name = model_name self._output_format = output_format - self._voice_id = voice_id self._params = params - self._settings = { - "model": model_name, - "voice_id": voice_id, - "output_format": output_format, - "language": str(params.language) if params.language else "en", - "speed": params.speed, - "sample_rate": sample_rate, - } + self._settings = GroqTTSSettings( + model=model_name, + voice=voice_id, + language=str(params.language) if params.language else "en", + output_format=output_format, + speed=params.speed, + groq_sample_rate=sample_rate, + ) + self._sync_model_name_to_metrics() self._client = AsyncGroq(api_key=self._api_key) @@ -129,8 +149,8 @@ class GroqTTSService(TTSService): try: response = await self._client.audio.speech.create( - model=self._model_name, - voice=self._voice_id, + model=self._settings.model, + voice=self._settings.voice, response_format=self._output_format, input=text, ) diff --git a/src/pipecat/services/hathora/stt.py b/src/pipecat/services/hathora/stt.py index defdc355d..a08a80aa2 100644 --- a/src/pipecat/services/hathora/stt.py +++ b/src/pipecat/services/hathora/stt.py @@ -8,6 +8,7 @@ import base64 import os +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional import aiohttp @@ -18,6 +19,7 @@ from pipecat.frames.frames import ( Frame, TranscriptionFrame, ) +from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven from pipecat.services.stt_latency import HATHORA_TTFS_P99 from pipecat.services.stt_service import SegmentedSTTService from pipecat.transcriptions.language import Language @@ -27,12 +29,27 @@ from pipecat.utils.tracing.service_decorators import traced_stt from .utils import ConfigOption +@dataclass +class HathoraSTTSettings(STTSettings): + """Settings for the Hathora STT service. + + Parameters: + config: Some models support additional config, refer to + `docs `_ for each model to see + what is supported. + """ + + config: list[ConfigOption] | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class HathoraSTTService(SegmentedSTTService): """This service supports several different speech-to-text models hosted by Hathora. [Documentation](https://models.hathora.dev) """ + _settings: HathoraSTTSettings + class InputParams(BaseModel): """Optional input parameters for Hathora STT configuration. @@ -83,12 +100,12 @@ class HathoraSTTService(SegmentedSTTService): params = params or HathoraSTTService.InputParams() - self._settings = { - "language": params.language, - "config": params.config, - } - - self.set_model_name(model) + self._settings = HathoraSTTSettings( + model=model, + language=params.language, + config=params.config, + ) + self._sync_model_name_to_metrics() def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -123,12 +140,11 @@ class HathoraSTTService(SegmentedSTTService): "model": self._model, } - if self._settings["language"] is not None: - payload["language"] = self._settings["language"] - if self._settings["config"] is not None: + if self._settings.language is not None: + payload["language"] = self._settings.language + if self._settings.config is not None: payload["model_config"] = [ - {"name": option.name, "value": option.value} - for option in self._settings["config"] + {"name": option.name, "value": option.value} for option in self._settings.config ] base64_audio = base64.b64encode(audio).decode("utf-8") @@ -147,7 +163,7 @@ class HathoraSTTService(SegmentedSTTService): if text: # Only yield non-empty text # Hathora's API currently doesn't return language info # so we default to the requested language or "en" - response_language = self._settings["language"] or "en" + response_language = self._settings.language or "en" await self._handle_transcription(text, True, response_language) yield TranscriptionFrame( text, diff --git a/src/pipecat/services/hathora/tts.py b/src/pipecat/services/hathora/tts.py index 80cbd4fe8..1e7662aab 100644 --- a/src/pipecat/services/hathora/tts.py +++ b/src/pipecat/services/hathora/tts.py @@ -9,6 +9,7 @@ import io import os import wave +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional, Tuple import aiohttp @@ -21,6 +22,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven from pipecat.services.tts_service import TTSService from pipecat.utils.tracing.service_decorators import traced_tts @@ -45,12 +47,29 @@ def _decode_audio_payload( return audio_bytes, fallback_sample_rate, fallback_channels +@dataclass +class HathoraTTSSettings(TTSSettings): + """Settings for Hathora TTS service. + + Parameters: + speed: Speech speed multiplier (if supported by model). + config: Some models support additional config, refer to + [docs](https://models.hathora.dev) for each model to see + what is supported. + """ + + speed: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + config: list[ConfigOption] | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class HathoraTTSService(TTSService): """This service supports several different text-to-speech models hosted by Hathora. [Documentation](https://models.hathora.dev) """ + _settings: HathoraTTSSettings + class InputParams(BaseModel): """Optional input parameters for Hathora TTS configuration. @@ -98,13 +117,13 @@ class HathoraTTSService(TTSService): params = params or HathoraTTSService.InputParams() - self._settings = { - "speed": params.speed, - "config": params.config, - } - - self.set_model_name(model) - self.set_voice(voice_id) + self._settings = HathoraTTSSettings( + model=model, + voice=voice_id, + speed=params.speed, + config=params.config, + ) + self._sync_model_name_to_metrics() def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -133,14 +152,13 @@ class HathoraTTSService(TTSService): payload = {"model": self._model, "text": text} - if self._voice_id is not None: - payload["voice"] = self._voice_id - if self._settings["speed"] is not None: - payload["speed"] = self._settings["speed"] - if self._settings["config"] is not None: + if self._settings.voice is not None: + payload["voice"] = self._settings.voice + if self._settings.speed is not None: + payload["speed"] = self._settings.speed + if self._settings.config is not None: payload["model_config"] = [ - {"name": option.name, "value": option.value} - for option in self._settings["config"] + {"name": option.name, "value": option.value} for option in self._settings.config ] yield TTSStartedFrame(context_id=context_id) diff --git a/src/pipecat/services/hume/tts.py b/src/pipecat/services/hume/tts.py index 2d98e1f8c..d15f13ce1 100644 --- a/src/pipecat/services/hume/tts.py +++ b/src/pipecat/services/hume/tts.py @@ -6,6 +6,8 @@ import base64 import os +import warnings +from dataclasses import dataclass, field from typing import Any, AsyncGenerator, Optional import httpx @@ -24,6 +26,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven from pipecat.services.tts_service import WordTTSService from pipecat.utils.tracing.service_decorators import traced_tts @@ -46,6 +49,21 @@ DEFAULT_HEADERS = { } +@dataclass +class HumeTTSSettings(TTSSettings): + """Settings for Hume TTS service. + + Parameters: + description: Natural-language acting directions (up to 100 characters). + speed: Speaking-rate multiplier (0.5-2.0). + trailing_silence: Seconds of silence to append at the end (0-5). + """ + + description: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + trailing_silence: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class HumeTTSService(WordTTSService): """Hume Octave Text-to-Speech service. @@ -61,6 +79,8 @@ class HumeTTSService(WordTTSService): - Provides metrics for Time To First Byte (TTFB) and TTS usage. """ + _settings: HumeTTSSettings + class InputParams(BaseModel): """Optional synthesis parameters for Hume TTS. @@ -114,10 +134,14 @@ class HumeTTSService(WordTTSService): self._http_client = httpx.AsyncClient(headers=DEFAULT_HEADERS) self._client = AsyncHumeClient(api_key=api_key, httpx_client=self._http_client) - self._params = params or HumeTTSService.InputParams() - # Store voice in the base class (mirrors other services) - self.set_voice(voice_id) + params = params or HumeTTSService.InputParams() + self._settings = HumeTTSSettings( + voice=voice_id, + description=params.description, + speed=params.speed, + trailing_silence=params.trailing_silence, + ) self._audio_bytes = b"" @@ -183,7 +207,10 @@ class HumeTTSService(WordTTSService): await self.add_word_timestamps([("Reset", 0)]) async def update_setting(self, key: str, value: Any) -> None: - """Runtime updates via `TTSUpdateSettingsFrame`. + """Runtime updates via key/value pair. + + .. deprecated:: 0.0.103 + Use ``TTSUpdateSettingsFrame(update=HumeTTSSettings(...))`` instead. Args: key: The name of the setting to update. Recognized keys are: @@ -193,20 +220,29 @@ class HumeTTSService(WordTTSService): - "trailing_silence" value: The new value for the setting. """ - key_l = (key or "").lower() + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "'update_setting' is deprecated, use " + "'TTSUpdateSettingsFrame(update=HumeTTSSettings(...))' instead.", + DeprecationWarning, + stacklevel=2, + ) - if key_l == "voice_id": - self.set_voice(str(value)) - logger.debug(f"HumeTTSService voice_id set to: {self.voice}") - elif key_l == "description": - self._params.description = None if value is None else str(value) - elif key_l == "speed": - self._params.speed = None if value is None else float(value) - elif key_l == "trailing_silence": - self._params.trailing_silence = None if value is None else float(value) - else: - # Defer unknown keys to the base class - await super().update_setting(key, value) + key_l = (key or "").lower() + known_keys = {"voice_id", "voice", "description", "speed", "trailing_silence"} + + if key_l in known_keys: + kwargs: dict[str, Any] = {} + if key_l in ("voice_id", "voice"): + kwargs["voice"] = str(value) + elif key_l == "description": + kwargs["description"] = None if value is None else str(value) + elif key_l == "speed": + kwargs["speed"] = None if value is None else float(value) + elif key_l == "trailing_silence": + kwargs["trailing_silence"] = None if value is None else float(value) + await self._update_settings(HumeTTSSettings(**kwargs)) @traced_tts async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]: @@ -226,14 +262,14 @@ class HumeTTSService(WordTTSService): # Build the request payload utterance_kwargs: dict[str, Any] = { "text": text, - "voice": PostedUtteranceVoiceWithId(id=self._voice_id), + "voice": PostedUtteranceVoiceWithId(id=self._settings.voice), } - if self._params.description is not None: - utterance_kwargs["description"] = self._params.description - if self._params.speed is not None: - utterance_kwargs["speed"] = self._params.speed - if self._params.trailing_silence is not None: - utterance_kwargs["trailing_silence"] = self._params.trailing_silence + if self._settings.description is not None: + utterance_kwargs["description"] = self._settings.description + if self._settings.speed is not None: + utterance_kwargs["speed"] = self._settings.speed + if self._settings.trailing_silence is not None: + utterance_kwargs["trailing_silence"] = self._settings.trailing_silence utterance = PostedUtterance(**utterance_kwargs) @@ -257,7 +293,7 @@ class HumeTTSService(WordTTSService): # Use version "2" by default if no description is provided # Version "1" is needed when description is used - version = "1" if self._params.description is not None else "2" + version = "1" if self._settings.description is not None else "2" # Track the duration of this utterance based on the last timestamp utterance_duration = 0.0 diff --git a/src/pipecat/services/inworld/tts.py b/src/pipecat/services/inworld/tts.py index adf72ce9f..11e6125c0 100644 --- a/src/pipecat/services/inworld/tts.py +++ b/src/pipecat/services/inworld/tts.py @@ -17,7 +17,8 @@ import asyncio import base64 import json import uuid -from typing import Any, AsyncGenerator, Dict, List, Literal, Optional, Tuple +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, ClassVar, Dict, List, Literal, Mapping, Optional, Tuple import aiohttp import websockets @@ -28,6 +29,8 @@ from pipecat import version as pipecat_version USER_AGENT = f"pipecat/{pipecat_version()}" from pydantic import BaseModel +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven, is_given + try: from websockets.asyncio.client import connect as websocket_connect from websockets.protocol import State @@ -52,6 +55,53 @@ from pipecat.services.tts_service import AudioContextWordTTSService, WordTTSServ from pipecat.utils.tracing.service_decorators import traced_tts +@dataclass +class InworldTTSSettings(TTSSettings): + """Settings for Inworld TTS services. + + Parameters: + audio_encoding: Audio encoding format (e.g. LINEAR16). + audio_sample_rate: Audio sample rate in Hz. + speaking_rate: Speaking rate for speech synthesis. + temperature: Temperature for speech synthesis. + auto_mode: Whether to use auto mode. Recommended when texts are sent + in full sentences/phrases. When enabled, the server controls + flushing of buffered text to achieve minimal latency while + maintaining high quality audio output. If None (default), + automatically set based on aggregate_sentences. + apply_text_normalization: Whether to apply text normalization. + timestamp_transport_strategy: Strategy for timestamp transport ("ASYNC" or "SYNC"). + """ + + audio_encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + audio_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + speaking_rate: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + temperature: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + auto_mode: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + apply_text_normalization: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + timestamp_transport_strategy: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + _aliases: ClassVar[Dict[str, str]] = { + "voice_id": "voice", + "voiceId": "voice", + "modelId": "model", + "applyTextNormalization": "apply_text_normalization", + "autoMode": "auto_mode", + "timestampTransportStrategy": "timestamp_transport_strategy", + } + + @classmethod + def from_mapping(cls, settings: Mapping[str, Any]) -> "InworldTTSSettings": + """Construct settings from a plain dict, destructuring legacy nested ``audioConfig``.""" + flat = dict(settings) + nested = flat.pop("audioConfig", None) + if isinstance(nested, dict): + flat.setdefault("audio_encoding", nested.get("audioEncoding")) + flat.setdefault("audio_sample_rate", nested.get("sampleRateHertz")) + flat.setdefault("speaking_rate", nested.get("speakingRate")) + return super().from_mapping(flat) + + class InworldHttpTTSService(WordTTSService): """Inworld AI HTTP-based TTS service. @@ -59,6 +109,8 @@ class InworldHttpTTSService(WordTTSService): Outputs LINEAR16 audio at configurable sample rates with word-level timestamps. """ + _settings: InworldTTSSettings + class InputParams(BaseModel): """Input parameters for Inworld TTS configuration. @@ -117,26 +169,23 @@ class InworldHttpTTSService(WordTTSService): else: self._base_url = "https://api.inworld.ai/tts/v1/voice" - self._settings = { - "voiceId": voice_id, - "modelId": model, - "audioConfig": { - "audioEncoding": encoding, - "sampleRateHertz": 0, - }, - } + self._settings = InworldTTSSettings( + model=model, + voice=voice_id, + audio_encoding=encoding, + audio_sample_rate=0, + ) if params.temperature is not None: - self._settings["temperature"] = params.temperature + self._settings.temperature = params.temperature if params.speaking_rate is not None: - self._settings["audioConfig"]["speakingRate"] = params.speaking_rate + self._settings.speaking_rate = params.speaking_rate if params.timestamp_transport_strategy is not None: - self._settings["timestampTransportStrategy"] = params.timestamp_transport_strategy + self._settings.timestamp_transport_strategy = params.timestamp_transport_strategy self._cumulative_time = 0.0 - self.set_voice(voice_id) - self.set_model_name(model) + self._sync_model_name_to_metrics() def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -153,7 +202,7 @@ class InworldHttpTTSService(WordTTSService): frame: The start frame. """ await super().start(frame) - self._settings["audioConfig"]["sampleRateHertz"] = self.sample_rate + self._settings.audio_sample_rate = self.sample_rate async def stop(self, frame: EndFrame): """Stop the Inworld TTS service. @@ -232,20 +281,27 @@ class InworldHttpTTSService(WordTTSService): """ logger.debug(f"{self}: Generating TTS [{text}] (streaming={self._streaming})") + audio_config = { + "audioEncoding": self._settings.audio_encoding, + "sampleRateHertz": self._settings.audio_sample_rate, + } + if is_given(self._settings.speaking_rate): + audio_config["speakingRate"] = self._settings.speaking_rate + payload = { "text": text, - "voiceId": self._settings["voiceId"], - "modelId": self._settings["modelId"], - "audioConfig": self._settings["audioConfig"], + "voiceId": self._settings.voice, + "modelId": self._settings.model, + "audioConfig": audio_config, } - if "temperature" in self._settings: - payload["temperature"] = self._settings["temperature"] + if is_given(self._settings.temperature): + payload["temperature"] = self._settings.temperature # Use WORD timestamps for simplicity and correct spacing/capitalization payload["timestampType"] = self._timestamp_type - if "timestampTransportStrategy" in self._settings: - payload["timestampTransportStrategy"] = self._settings["timestampTransportStrategy"] + if is_given(self._settings.timestamp_transport_strategy): + payload["timestampTransportStrategy"] = self._settings.timestamp_transport_strategy request_id = str(uuid.uuid4()) headers = { @@ -419,6 +475,8 @@ class InworldTTSService(AudioContextWordTTSService): with word-level timestamps. """ + _settings: InworldTTSSettings + class InputParams(BaseModel): """Input parameters for Inworld WebSocket TTS configuration. @@ -486,29 +544,27 @@ class InworldTTSService(AudioContextWordTTSService): self._api_key = api_key self._url = url - self._settings: Dict[str, Any] = { - "voiceId": voice_id, - "modelId": model, - "audioConfig": { - "audioEncoding": encoding, - "sampleRateHertz": 0, - }, - } + self._settings = InworldTTSSettings( + model=model, + voice=voice_id, + audio_encoding=encoding, + audio_sample_rate=0, + ) self._timestamp_type = "WORD" if params.temperature is not None: - self._settings["temperature"] = params.temperature + self._settings.temperature = params.temperature if params.speaking_rate is not None: - self._settings["audioConfig"]["speakingRate"] = params.speaking_rate + self._settings.speaking_rate = params.speaking_rate if params.apply_text_normalization is not None: - self._settings["applyTextNormalization"] = params.apply_text_normalization + self._settings.apply_text_normalization = params.apply_text_normalization if params.timestamp_transport_strategy is not None: - self._settings["timestampTransportStrategy"] = params.timestamp_transport_strategy + self._settings.timestamp_transport_strategy = params.timestamp_transport_strategy if params.auto_mode is not None: - self._settings["autoMode"] = params.auto_mode + self._settings.auto_mode = params.auto_mode else: - self._settings["autoMode"] = aggregate_sentences + self._settings.auto_mode = aggregate_sentences self._buffer_settings = { "maxBufferDelayMs": params.max_buffer_delay_ms, @@ -526,8 +582,7 @@ class InworldTTSService(AudioContextWordTTSService): # Track the end time of the last word in the current generation self._generation_end_time = 0.0 - self.set_voice(voice_id) - self.set_model_name(model) + self._sync_model_name_to_metrics() def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -544,7 +599,7 @@ class InworldTTSService(AudioContextWordTTSService): frame: The start frame. """ await super().start(frame) - self._settings["audioConfig"]["sampleRateHertz"] = self.sample_rate + self._settings.audio_sample_rate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): @@ -700,6 +755,21 @@ class InworldTTSService(AudioContextWordTTSService): await self._disconnect_websocket() + async def _update_settings(self, update: TTSSettings) -> dict[str, Any]: + """Apply a settings update. + + Settings are stored but not applied to the active connection. + """ + changed = await super()._update_settings(update) + + if not changed: + return changed + + await self._disconnect() + await self._connect() + + return changed + async def _connect_websocket(self): """Connect to the Inworld WebSocket TTS service. @@ -883,22 +953,29 @@ class InworldTTSService(AudioContextWordTTSService): Args: context_id: The context ID. """ + audio_config = { + "audioEncoding": self._settings.audio_encoding, + "sampleRateHertz": self._settings.audio_sample_rate, + } + if is_given(self._settings.speaking_rate): + audio_config["speakingRate"] = self._settings.speaking_rate + create_config: Dict[str, Any] = { - "voiceId": self._settings["voiceId"], - "modelId": self._settings["modelId"], - "audioConfig": self._settings["audioConfig"], + "voiceId": self._settings.voice, + "modelId": self._settings.model, + "audioConfig": audio_config, } - if "temperature" in self._settings: - create_config["temperature"] = self._settings["temperature"] - if "applyTextNormalization" in self._settings: - create_config["applyTextNormalization"] = self._settings["applyTextNormalization"] - if "autoMode" in self._settings: - create_config["autoMode"] = self._settings["autoMode"] - if "timestampTransportStrategy" in self._settings: - create_config["timestampTransportStrategy"] = self._settings[ - "timestampTransportStrategy" - ] + if is_given(self._settings.temperature): + create_config["temperature"] = self._settings.temperature + if is_given(self._settings.apply_text_normalization): + create_config["applyTextNormalization"] = self._settings.apply_text_normalization + if is_given(self._settings.auto_mode): + create_config["autoMode"] = self._settings.auto_mode + if is_given(self._settings.timestamp_transport_strategy): + create_config["timestampTransportStrategy"] = ( + self._settings.timestamp_transport_strategy + ) # Set buffer settings for timely audio generation. # Use provided values or defaults that work well for streaming LLM output. diff --git a/src/pipecat/services/kokoro/tts.py b/src/pipecat/services/kokoro/tts.py index 49ede2409..6e848ae87 100644 --- a/src/pipecat/services/kokoro/tts.py +++ b/src/pipecat/services/kokoro/tts.py @@ -7,6 +7,7 @@ """Kokoro TTS service implementation using kokoro-onnx.""" import os +from dataclasses import dataclass, field from pathlib import Path from typing import AsyncGenerator, Optional @@ -22,6 +23,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven from pipecat.services.tts_service import TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -87,6 +89,17 @@ def language_to_kokoro_language(language: Language) -> str: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +@dataclass +class KokoroTTSSettings(TTSSettings): + """Settings for the Kokoro TTS service. + + Parameters: + lang_code: Kokoro language code for synthesis. + """ + + lang_code: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class KokoroTTSService(TTSService): """Kokoro TTS service implementation. @@ -94,6 +107,8 @@ class KokoroTTSService(TTSService): Automatically downloads model files on first use. """ + _settings: KokoroTTSSettings + class InputParams(BaseModel): """Input parameters for Kokoro TTS configuration. @@ -126,9 +141,14 @@ class KokoroTTSService(TTSService): params = params or KokoroTTSService.InputParams() - self._voice_id = voice_id self._lang_code = language_to_kokoro_language(params.language) + self._settings = KokoroTTSSettings( + voice=voice_id, + language=language_to_kokoro_language(params.language), + lang_code=language_to_kokoro_language(params.language), + ) + model = Path(model_path) if model_path else KOKORO_CACHE_DIR / "kokoro-v1.0.onnx" voices = Path(voices_path) if voices_path else KOKORO_CACHE_DIR / "voices-v1.0.bin" @@ -161,7 +181,7 @@ class KokoroTTSService(TTSService): yield TTSStartedFrame(context_id=context_id) stream = self._kokoro.create_stream( - text, voice=self._voice_id, lang=self._lang_code, speed=1.0 + text, voice=self._settings.voice, lang=self._lang_code, speed=1.0 ) async for samples, sample_rate in stream: diff --git a/src/pipecat/services/llm_service.py b/src/pipecat/services/llm_service.py index c8af00b80..e4f1adb45 100644 --- a/src/pipecat/services/llm_service.py +++ b/src/pipecat/services/llm_service.py @@ -44,6 +44,7 @@ from pipecat.frames.frames import ( LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMTextFrame, + LLMUpdateSettingsFrame, StartFrame, UserImageRequestFrame, ) @@ -58,6 +59,7 @@ from pipecat.processors.aggregators.llm_response import ( from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_service import AIService +from pipecat.services.settings import LLMSettings, is_given from pipecat.turns.user_turn_completion_mixin import UserTurnCompletionLLMServiceMixin from pipecat.utils.context.llm_context_summarization import ( LLMContextSummarizationUtil, @@ -172,6 +174,8 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService): logger.info(f"Starting {len(function_calls)} function calls") """ + _settings: LLMSettings + # OpenAILLMAdapter is used as the default adapter since it aligns with most LLM implementations. # However, subclasses should override this with a more specific adapter when necessary. adapter_class: Type[BaseLLMAdapter] = OpenAILLMAdapter @@ -200,6 +204,7 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService): self._sequential_runner_task: Optional[asyncio.Task] = None self._skip_tts: Optional[bool] = None self._summary_task: Optional[asyncio.Task] = None + self._settings = LLMSettings() # Here in case subclass doesn't implement more specific settings (hopefully shouldn't happen) self._register_event_handler("on_function_calls_started") self._register_event_handler("on_completion_timeout") @@ -307,34 +312,28 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService): await self._cancel_sequential_runner_task() await self._cancel_summary_task() - async def _update_settings(self, settings: Mapping[str, Any]): - """Update LLM service settings. - - Handles turn completion settings specially since they are not model - parameters and should not be passed to the underlying LLM API. + async def _update_settings(self, update: LLMSettings) -> dict[str, Any]: + """Apply a settings update, handling turn-completion fields. Args: - settings: Dictionary of settings to update. - """ - # Turn completion settings to extract (not model parameters) - turn_completion_keys = {"filter_incomplete_user_turns", "user_turn_completion_config"} + update: An LLM settings delta. - # Handle turn completion settings - if "filter_incomplete_user_turns" in settings: - self._filter_incomplete_user_turns = settings["filter_incomplete_user_turns"] + Returns: + Dict mapping changed field names to their previous values. + """ + changed = await super()._update_settings(update) + + if "filter_incomplete_user_turns" in changed: + self._filter_incomplete_user_turns = self._settings.filter_incomplete_user_turns logger.info( - f"{self}: Incomplete turn filtering {'enabled' if self._filter_incomplete_user_turns else 'disabled'}" + f"{self}: Incomplete turn filtering " + f"{'enabled' if self._filter_incomplete_user_turns else 'disabled'}" ) - # Configure the mixin with config object - if self._filter_incomplete_user_turns and "user_turn_completion_config" in settings: - self.set_user_turn_completion_config(settings["user_turn_completion_config"]) + if "user_turn_completion_config" in changed and self._filter_incomplete_user_turns: + self.set_user_turn_completion_config(self._settings.user_turn_completion_config) - # Remove turn completion settings before passing to parent - settings = {k: v for k, v in settings.items() if k not in turn_completion_keys} - - # Let the parent handle remaining model parameters - await super()._update_settings(settings) + return changed async def process_frame(self, frame: Frame, direction: FrameDirection): """Process a frame. @@ -349,6 +348,21 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService): await self._handle_interruptions(frame) elif isinstance(frame, LLMConfigureOutputFrame): self._skip_tts = frame.skip_tts + elif isinstance(frame, LLMUpdateSettingsFrame): + if frame.update is not None: + await self._update_settings(frame.update) + elif frame.settings: + # Backward-compatible path: convert legacy dict to settings object. + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Passing a dict via LLMUpdateSettingsFrame(settings={...}) is deprecated " + "since 0.0.103, use LLMUpdateSettingsFrame(update=LLMSettings(...)) instead.", + DeprecationWarning, + stacklevel=2, + ) + update = type(self._settings).from_mapping(frame.settings) + await self._update_settings(update) elif isinstance(frame, LLMContextSummaryRequestFrame): await self._handle_summary_request(frame) diff --git a/src/pipecat/services/lmnt/tts.py b/src/pipecat/services/lmnt/tts.py index 4c34e28d5..f70bfa402 100644 --- a/src/pipecat/services/lmnt/tts.py +++ b/src/pipecat/services/lmnt/tts.py @@ -7,7 +7,8 @@ """LMNT text-to-speech service implementation.""" import json -from typing import AsyncGenerator, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, Optional from loguru import logger @@ -23,6 +24,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven from pipecat.services.tts_service import InterruptibleTTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -71,6 +73,17 @@ def language_to_lmnt_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +@dataclass +class LmntTTSSettings(TTSSettings): + """Settings for LMNT TTS service. + + Parameters: + format: Audio output format. Defaults to "raw". + """ + + format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class LmntTTSService(InterruptibleTTSService): """LMNT real-time text-to-speech service. @@ -79,6 +92,8 @@ class LmntTTSService(InterruptibleTTSService): language settings. """ + _settings: LmntTTSSettings + def __init__( self, *, @@ -107,12 +122,13 @@ class LmntTTSService(InterruptibleTTSService): ) self._api_key = api_key - self.set_voice(voice_id) - self.set_model_name(model) - self._settings = { - "language": self.language_to_service_language(language), - "format": "raw", # Use raw format for direct PCM data - } + self._settings = LmntTTSSettings( + model=model, + voice=voice_id, + language=self.language_to_service_language(language), + format="raw", + ) + self._sync_model_name_to_metrics() self._receive_task = None self._context_id: Optional[str] = None @@ -190,6 +206,23 @@ class LmntTTSService(InterruptibleTTSService): await self._disconnect_websocket() + async def _update_settings(self, update: TTSSettings) -> dict[str, Any]: + """Apply a settings update. + + Args: + update: A :class:`TTSSettings` (or ``LmntTTSSettings``) delta. + + Returns: + Dict mapping changed field names to their previous values. + """ + changed = await super()._update_settings(update) + + if changed: + await self._disconnect() + await self._connect() + + return changed + async def _connect_websocket(self): """Connect to LMNT websocket.""" try: @@ -201,11 +234,11 @@ class LmntTTSService(InterruptibleTTSService): # Build initial connection message init_msg = { "X-API-Key": self._api_key, - "voice": self._voice_id, - "format": self._settings["format"], + "voice": self._settings.voice, + "format": self._settings.format, "sample_rate": self.sample_rate, - "language": self._settings["language"], - "model": self.model_name, + "language": self._settings.language, + "model": self._settings.model, } # Connect to LMNT's websocket directly diff --git a/src/pipecat/services/minimax/tts.py b/src/pipecat/services/minimax/tts.py index 7284d9630..54925f7e4 100644 --- a/src/pipecat/services/minimax/tts.py +++ b/src/pipecat/services/minimax/tts.py @@ -11,7 +11,8 @@ for streaming text-to-speech synthesis. """ import json -from typing import AsyncGenerator, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, ClassVar, Dict, Mapping, Optional import aiohttp from loguru import logger @@ -25,6 +26,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven, is_given from pipecat.services.tts_service import TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -85,6 +87,69 @@ def language_to_minimax_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +@dataclass +class MiniMaxTTSSettings(TTSSettings): + """Settings for MiniMax TTS service. + + Parameters: + stream: Whether to use streaming mode. + speed: Speech speed (range: 0.5 to 2.0). + volume: Speech volume (range: 0 to 10). + pitch: Pitch adjustment (range: -12 to 12). + emotion: Emotional tone (options: "happy", "sad", "angry", "fearful", + "disgusted", "surprised", "calm", "fluent"). + text_normalization: Enable text normalization (Chinese/English). + latex_read: Enable LaTeX formula reading. + audio_bitrate: Audio bitrate in bps. + audio_format: Audio output format. + audio_channel: Number of audio channels. + audio_sample_rate: Audio sample rate in Hz. + language_boost: Language boost string for multilingual support. + """ + + stream: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + volume: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + pitch: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + emotion: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + text_normalization: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + latex_read: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + audio_bitrate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + audio_format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + audio_channel: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + audio_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + language_boost: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + _aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice"} + + @classmethod + def from_mapping(cls, settings: Mapping[str, Any]) -> "MiniMaxTTSSettings": + """Construct settings from a plain dict, destructuring legacy nested dicts. + + Handles ``voice_setting`` (with ``vol`` → ``volume`` rename) and + ``audio_setting`` (with prefixed field mapping). + """ + flat = dict(settings) + + voice = flat.pop("voice_setting", None) + if isinstance(voice, dict): + flat.setdefault("speed", voice.get("speed")) + flat.setdefault("volume", voice.get("vol")) + flat.setdefault("pitch", voice.get("pitch")) + flat.setdefault("emotion", voice.get("emotion")) + flat.setdefault("text_normalization", voice.get("text_normalization")) + flat.setdefault("latex_read", voice.get("latex_read")) + + audio = flat.pop("audio_setting", None) + if isinstance(audio, dict): + flat.setdefault("audio_bitrate", audio.get("bitrate")) + flat.setdefault("audio_format", audio.get("format")) + flat.setdefault("audio_channel", audio.get("channel")) + flat.setdefault("audio_sample_rate", audio.get("sample_rate")) + + return super().from_mapping(flat) + + class MiniMaxHttpTTSService(TTSService): """Text-to-speech service using MiniMax's T2A (Text-to-Audio) API. @@ -96,6 +161,8 @@ class MiniMaxHttpTTSService(TTSService): https://www.minimax.io/platform/document/T2A%20V2?key=66719005a427f0c8a5701643 """ + _settings: MiniMaxTTSSettings + class InputParams(BaseModel): """Configuration parameters for MiniMax TTS. @@ -168,33 +235,26 @@ class MiniMaxHttpTTSService(TTSService): self._group_id = group_id self._base_url = f"{base_url}?GroupId={group_id}" self._session = aiohttp_session - self._model_name = model - self._voice_id = voice_id # Create voice settings - self._settings = { - "stream": True, - "voice_setting": { - "speed": params.speed, - "vol": params.volume, - "pitch": params.pitch, - }, - "audio_setting": { - "bitrate": 128000, - "format": "pcm", - "channel": 1, - }, - } - - # Set voice and model - self.set_voice(voice_id) - self.set_model_name(model) + self._settings = MiniMaxTTSSettings( + model=model, + voice=voice_id, + stream=True, + speed=params.speed, + volume=params.volume, + pitch=params.pitch, + audio_bitrate=128000, + audio_format="pcm", + audio_channel=1, + ) + self._sync_model_name_to_metrics() # Add language boost if provided if params.language: service_lang = self.language_to_service_language(params.language) if service_lang: - self._settings["language_boost"] = service_lang + self._settings.language_boost = service_lang # Add optional emotion if provided if params.emotion: @@ -210,7 +270,7 @@ class MiniMaxHttpTTSService(TTSService): "fluent", ] if params.emotion in supported_emotions: - self._settings["voice_setting"]["emotion"] = params.emotion + self._settings.emotion = params.emotion else: logger.warning( f"Unsupported emotion: {params.emotion}. Supported emotions: {supported_emotions}" @@ -226,15 +286,15 @@ class MiniMaxHttpTTSService(TTSService): "Parameter `english_normalization` is deprecated and will be removed in a future version. Use `text_normalization` instead.", DeprecationWarning, ) - self._settings["voice_setting"]["text_normalization"] = params.english_normalization + self._settings.text_normalization = params.english_normalization # Add text_normalization if provided (corrected parameter name) if params.text_normalization is not None: - self._settings["voice_setting"]["text_normalization"] = params.text_normalization + self._settings.text_normalization = params.text_normalization # Add latex_read if provided if params.latex_read is not None: - self._settings["voice_setting"]["latex_read"] = params.latex_read + self._settings.latex_read = params.latex_read def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -255,24 +315,6 @@ class MiniMaxHttpTTSService(TTSService): """ return language_to_minimax_language(language) - def set_model_name(self, model: str): - """Set the TTS model to use. - - Args: - model: The model name to use for synthesis. - """ - self._model_name = model - - def set_voice(self, voice: str): - """Set the voice to use. - - Args: - voice: The voice identifier to use for synthesis. - """ - self._voice_id = voice - if "voice_setting" in self._settings: - self._settings["voice_setting"]["voice_id"] = voice - async def start(self, frame: StartFrame): """Start the MiniMax TTS service. @@ -280,7 +322,7 @@ class MiniMaxHttpTTSService(TTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["audio_setting"]["sample_rate"] = self.sample_rate + self._settings.audio_sample_rate = self.sample_rate logger.debug(f"MiniMax TTS initialized with sample_rate: {self.sample_rate}") @traced_tts @@ -302,10 +344,38 @@ class MiniMaxHttpTTSService(TTSService): "Authorization": f"Bearer {self._api_key}", } + # Build voice_setting dict for API + voice_setting = { + "voice_id": self._settings.voice, + "speed": self._settings.speed, + "vol": self._settings.volume, + "pitch": self._settings.pitch, + } + if is_given(self._settings.emotion): + voice_setting["emotion"] = self._settings.emotion + if is_given(self._settings.text_normalization): + voice_setting["text_normalization"] = self._settings.text_normalization + if is_given(self._settings.latex_read): + voice_setting["latex_read"] = self._settings.latex_read + + # Build audio_setting dict for API + audio_setting = { + "bitrate": self._settings.audio_bitrate, + "format": self._settings.audio_format, + "channel": self._settings.audio_channel, + "sample_rate": self._settings.audio_sample_rate, + } + # Create payload from settings - payload = self._settings.copy() - payload["model"] = self._model_name - payload["text"] = text + payload = { + "stream": self._settings.stream, + "voice_setting": voice_setting, + "audio_setting": audio_setting, + "model": self._settings.model, + "text": text, + } + if is_given(self._settings.language_boost): + payload["language_boost"] = self._settings.language_boost try: await self.start_ttfb_metrics() diff --git a/src/pipecat/services/mistral/llm.py b/src/pipecat/services/mistral/llm.py index 54361ef28..984ffb7dd 100644 --- a/src/pipecat/services/mistral/llm.py +++ b/src/pipecat/services/mistral/llm.py @@ -180,24 +180,24 @@ class MistralLLMService(OpenAILLMService): fixed_messages = self._apply_mistral_fixups(params_from_context["messages"]) params = { - "model": self.model_name, + "model": self._settings.model, "stream": True, "messages": fixed_messages, "tools": params_from_context["tools"], "tool_choice": params_from_context["tool_choice"], - "frequency_penalty": self._settings["frequency_penalty"], - "presence_penalty": self._settings["presence_penalty"], - "temperature": self._settings["temperature"], - "top_p": self._settings["top_p"], - "max_tokens": self._settings["max_tokens"], + "frequency_penalty": self._settings.frequency_penalty, + "presence_penalty": self._settings.presence_penalty, + "temperature": self._settings.temperature, + "top_p": self._settings.top_p, + "max_tokens": self._settings.max_tokens, } # Handle Mistral-specific parameter mapping # Mistral uses "random_seed" instead of "seed" - if self._settings["seed"]: - params["random_seed"] = self._settings["seed"] + if self._settings.seed: + params["random_seed"] = self._settings.seed # Add any extra parameters - params.update(self._settings["extra"]) + params.update(self._settings.extra) return params diff --git a/src/pipecat/services/moondream/vision.py b/src/pipecat/services/moondream/vision.py index 6a180b4cb..16be15ac5 100644 --- a/src/pipecat/services/moondream/vision.py +++ b/src/pipecat/services/moondream/vision.py @@ -81,7 +81,8 @@ class MoondreamService(VisionService): """ super().__init__(**kwargs) - self.set_model_name(model) + self._settings.model = model + self._sync_model_name_to_metrics() if not use_cpu: device, dtype = detect_device() diff --git a/src/pipecat/services/neuphonic/tts.py b/src/pipecat/services/neuphonic/tts.py index 24eb05bd3..dd2360e4c 100644 --- a/src/pipecat/services/neuphonic/tts.py +++ b/src/pipecat/services/neuphonic/tts.py @@ -13,7 +13,8 @@ text-to-speech API for real-time audio synthesis. import asyncio import base64 import json -from typing import Any, AsyncGenerator, Mapping, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, Optional import aiohttp from loguru import logger @@ -34,6 +35,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven from pipecat.services.tts_service import InterruptibleTTSService, TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -72,6 +74,21 @@ def language_to_neuphonic_lang_code(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +@dataclass +class NeuphonicTTSSettings(TTSSettings): + """Settings for Neuphonic TTS service. + + Parameters: + speed: Speech speed multiplier. Defaults to 1.0. + encoding: Audio encoding format. + sampling_rate: Audio sample rate. + """ + + speed: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + sampling_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class NeuphonicTTSService(InterruptibleTTSService): """Neuphonic real-time text-to-speech service using WebSocket streaming. @@ -80,6 +97,8 @@ class NeuphonicTTSService(InterruptibleTTSService): parameters for high-quality speech generation. """ + _settings: NeuphonicTTSSettings + class InputParams(BaseModel): """Input parameters for Neuphonic TTS configuration. @@ -127,13 +146,13 @@ class NeuphonicTTSService(InterruptibleTTSService): self._api_key = api_key self._url = url - self._settings = { - "lang_code": self.language_to_service_language(params.language), - "speed": params.speed, - "encoding": encoding, - "sampling_rate": sample_rate, - } - self.set_voice(voice_id) + self._settings = NeuphonicTTSSettings( + language=self.language_to_service_language(params.language), + speed=params.speed, + encoding=encoding, + sampling_rate=sample_rate, + voice=voice_id, + ) self._cumulative_time = 0 @@ -160,15 +179,14 @@ class NeuphonicTTSService(InterruptibleTTSService): """ return language_to_neuphonic_lang_code(language) - async def _update_settings(self, settings: Mapping[str, Any]): - """Update service settings and reconnect with new configuration.""" - if "voice_id" in settings: - self.set_voice(settings["voice_id"]) - - await super()._update_settings(settings) - await self._disconnect() - await self._connect() - logger.info(f"Switching TTS to settings: [{self._settings}]") + async def _update_settings(self, update: TTSSettings) -> dict[str, Any]: + """Apply a settings update and reconnect with new configuration.""" + changed = await super()._update_settings(update) + if changed: + await self._disconnect() + await self._connect() + logger.info(f"Switching TTS to settings: [{self._settings}]") + return changed async def start(self, frame: StartFrame): """Start the Neuphonic TTS service. @@ -266,8 +284,11 @@ class NeuphonicTTSService(InterruptibleTTSService): logger.debug("Connecting to Neuphonic") tts_config = { - **self._settings, - "voice_id": self._voice_id, + "lang_code": self._settings.language, + "speed": self._settings.speed, + "encoding": self._settings.encoding, + "sampling_rate": self._settings.sampling_rate, + "voice_id": self._settings.voice, } query_params = [] @@ -275,7 +296,7 @@ class NeuphonicTTSService(InterruptibleTTSService): if value is not None: query_params.append(f"{key}={value}") - url = f"{self._url}/speak/{self._settings['lang_code']}" + url = f"{self._url}/speak/{self._settings.language}" if query_params: url += f"?{'&'.join(query_params)}" @@ -384,6 +405,8 @@ class NeuphonicHttpTTSService(TTSService): HTTP-based communication over WebSocket connections. """ + _settings: NeuphonicTTSSettings + class InputParams(BaseModel): """Input parameters for Neuphonic HTTP TTS configuration. @@ -426,10 +449,13 @@ class NeuphonicHttpTTSService(TTSService): self._api_key = api_key self._session = aiohttp_session self._base_url = url.rstrip("/") - self._lang_code = self.language_to_service_language(params.language) or "en" - self._speed = params.speed - self._encoding = encoding - self.set_voice(voice_id) + self._settings = NeuphonicTTSSettings( + voice=voice_id, + language=self.language_to_service_language(params.language) or "en", + speed=params.speed, + encoding=encoding, + sampling_rate=sample_rate, + ) def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -513,7 +539,7 @@ class NeuphonicHttpTTSService(TTSService): """ logger.debug(f"Generating TTS: [{text}]") - url = f"{self._base_url}/sse/speak/{self._lang_code}" + url = f"{self._base_url}/sse/speak/{self._settings.language}" headers = { "X-API-KEY": self._api_key, @@ -522,14 +548,14 @@ class NeuphonicHttpTTSService(TTSService): payload = { "text": text, - "lang_code": self._lang_code, - "encoding": self._encoding, + "lang_code": self._settings.language, + "encoding": self._settings.encoding, "sampling_rate": self.sample_rate, - "speed": self._speed, + "speed": self._settings.speed, } - if self._voice_id: - payload["voice_id"] = self._voice_id + if self._settings.voice: + payload["voice_id"] = self._settings.voice try: await self.start_ttfb_metrics() diff --git a/src/pipecat/services/nvidia/stt.py b/src/pipecat/services/nvidia/stt.py index 8eb6d7bb5..fd924204e 100644 --- a/src/pipecat/services/nvidia/stt.py +++ b/src/pipecat/services/nvidia/stt.py @@ -8,7 +8,8 @@ import asyncio from concurrent.futures import CancelledError as FuturesCancelledError -from typing import AsyncGenerator, List, Mapping, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, List, Mapping, Optional from loguru import logger from pydantic import BaseModel @@ -22,6 +23,7 @@ from pipecat.frames.frames import ( StartFrame, TranscriptionFrame, ) +from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven from pipecat.services.stt_latency import NVIDIA_TTFS_P99 from pipecat.services.stt_service import SegmentedSTTService, STTService from pipecat.transcriptions.language import Language, resolve_language @@ -89,6 +91,32 @@ def language_to_nvidia_riva_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +@dataclass +class NvidiaSTTSettings(STTSettings): + """Settings for the NVIDIA Riva streaming STT service.""" + + pass + + +@dataclass +class NvidiaSegmentedSTTSettings(STTSettings): + """Settings for the NVIDIA Riva segmented STT service. + + Parameters: + profanity_filter: Whether to filter profanity from results. + automatic_punctuation: Whether to add automatic punctuation. + verbatim_transcripts: Whether to return verbatim transcripts. + boosted_lm_words: List of words to boost in language model. + boosted_lm_score: Score boost for specified words. + """ + + profanity_filter: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + automatic_punctuation: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + verbatim_transcripts: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + boosted_lm_words: List[str] | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + boosted_lm_score: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class NvidiaSTTService(STTService): """Real-time speech-to-text service using NVIDIA Riva streaming ASR. @@ -97,6 +125,8 @@ class NvidiaSTTService(STTService): processing for low-latency applications. """ + _settings: NvidiaSTTSettings + class InputParams(BaseModel): """Configuration parameters for NVIDIA Riva STT service. @@ -141,12 +171,6 @@ class NvidiaSTTService(STTService): self._server = server self._api_key = api_key self._use_ssl = use_ssl - self._profanity_filter = False - self._automatic_punctuation = True - self._no_verbatim_transcripts = False - self._language_code = params.language - self._boosted_lm_words = None - self._boosted_lm_score = 4.0 self._start_history = -1 self._start_threshold = -1.0 self._stop_history = -1 @@ -156,16 +180,11 @@ class NvidiaSTTService(STTService): self._custom_configuration = "" self._function_id = model_function_map.get("function_id") - self._settings = { - "language": str(params.language), - "profanity_filter": self._profanity_filter, - "automatic_punctuation": self._automatic_punctuation, - "verbatim_transcripts": not self._no_verbatim_transcripts, - "boosted_lm_words": self._boosted_lm_words, - "boosted_lm_score": self._boosted_lm_score, - } - - self.set_model_name(model_function_map.get("model_name")) + self._settings = NvidiaSTTSettings( + model=model_function_map.get("model_name"), + language=params.language, + ) + self._sync_model_name_to_metrics() self._asr_service = None self._queue = None @@ -186,22 +205,18 @@ class NvidiaSTTService(STTService): config = riva.client.StreamingRecognitionConfig( config=riva.client.RecognitionConfig( encoding=riva.client.AudioEncoding.LINEAR_PCM, - language_code=self._language_code, + language_code=self._settings.language, model="", max_alternatives=1, - profanity_filter=self._profanity_filter, - enable_automatic_punctuation=self._automatic_punctuation, - verbatim_transcripts=not self._no_verbatim_transcripts, + profanity_filter=False, + enable_automatic_punctuation=True, + verbatim_transcripts=True, sample_rate_hertz=self.sample_rate, audio_channel_count=1, ), interim_results=True, ) - riva.client.add_word_boosting_to_config( - config, self._boosted_lm_words, self._boosted_lm_score - ) - riva.client.add_endpoint_parameters_to_config( config, self._start_history, @@ -226,18 +241,31 @@ class NvidiaSTTService(STTService): async def set_model(self, model: str): """Set the ASR model for transcription. + .. deprecated:: 0.0.103 + Model cannot be changed after initialization for NVIDIA Riva streaming STT. + Set model and function id in the constructor instead, e.g.:: + + NvidiaSTTService( + api_key=..., + model_function_map={"function_id": "", "model_name": ""}, + ) + Args: model: Model name to set. - - Note: - Model cannot be changed after initialization. Use model_function_map - parameter in constructor instead. """ - logger.warning(f"Cannot set model after initialization. Set model and function id like so:") - example = {"function_id": "", "model_name": ""} - logger.warning( - f"{self.__class__.__name__}(api_key=, model_function_map={example})" - ) + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "'set_model' is deprecated. Model cannot be changed after initialization" + " for NVIDIA Riva streaming STT. Set model and function id in the" + " constructor instead, e.g.:" + " NvidiaSTTService(api_key=..., model_function_map=" + "{'function_id': '', 'model_name': ''})", + DeprecationWarning, + stacklevel=2, + ) async def start(self, frame: StartFrame): """Start the NVIDIA Riva STT service and initialize streaming configuration. @@ -254,7 +282,7 @@ class NvidiaSTTService(STTService): if not self._thread_task: self._thread_task = self.create_task(self._thread_task_handler()) - logger.debug(f"Initialized NvidiaSTTService with model: {self.model_name}") + logger.debug(f"Initialized NvidiaSTTService with model: {self._settings.model}") async def stop(self, frame: EndFrame): """Stop the NVIDIA Riva STT service and clean up resources. @@ -318,14 +346,14 @@ class NvidiaSTTService(STTService): transcript, self._user_id, time_now_iso8601(), - self._language_code, + self._settings.language, result=result, ) ) await self._handle_transcription( transcript=transcript, is_final=result.is_final, - language=self._language_code, + language=self._settings.language, ) else: await self.push_frame( @@ -333,7 +361,7 @@ class NvidiaSTTService(STTService): transcript, self._user_id, time_now_iso8601(), - self._language_code, + self._settings.language, result=result, ) ) @@ -386,6 +414,8 @@ class NvidiaSegmentedSTTService(SegmentedSTTService): audio buffering and speech detection. """ + _settings: NvidiaSegmentedSTTSettings + class InputParams(BaseModel): """Configuration parameters for NVIDIA Riva segmented STT service. @@ -437,26 +467,11 @@ class NvidiaSegmentedSTTService(SegmentedSTTService): params = params or NvidiaSegmentedSTTService.InputParams() - # Set model name - self.set_model_name(model_function_map.get("model_name")) - # Initialize NVIDIA Riva settings self._api_key = api_key self._server = server self._use_ssl = use_ssl self._function_id = model_function_map.get("function_id") - self._model_name = model_function_map.get("model_name") - - # Store the language as a Language enum and as a string - self._language_enum = params.language or Language.EN_US - self._language = self.language_to_service_language(self._language_enum) or "en-US" - - # Configure transcription parameters - self._profanity_filter = params.profanity_filter - self._automatic_punctuation = params.automatic_punctuation - self._verbatim_transcripts = params.verbatim_transcripts - self._boosted_lm_words = params.boosted_lm_words - self._boosted_lm_score = params.boosted_lm_score # Voice activity detection thresholds (use NVIDIA Riva defaults) self._start_history = -1 @@ -467,10 +482,19 @@ class NvidiaSegmentedSTTService(SegmentedSTTService): self._stop_threshold_eou = -1.0 self._custom_configuration = "" - # Create NVIDIA Riva client self._config = None self._asr_service = None - self._settings = {"language": self._language_enum} + self._settings = NvidiaSegmentedSTTSettings( + model=model_function_map.get("model_name"), + language=self.language_to_service_language(params.language or Language.EN_US) + or "en-US", + profanity_filter=params.profanity_filter, + automatic_punctuation=params.automatic_punctuation, + verbatim_transcripts=params.verbatim_transcripts, + boosted_lm_words=params.boosted_lm_words, + boosted_lm_score=params.boosted_lm_score, + ) + self._sync_model_name_to_metrics() def language_to_service_language(self, language: Language) -> Optional[str]: """Convert pipecat Language enum to NVIDIA Riva's language code. @@ -498,21 +522,25 @@ class NvidiaSegmentedSTTService(SegmentedSTTService): auth = riva.client.Auth(None, self._use_ssl, self._server, metadata) self._asr_service = riva.client.ASRService(auth) + def _get_language_code(self) -> str: + """Get the current NVIDIA Riva language code string.""" + return self._settings.language or "en-US" + def _create_recognition_config(self): """Create the NVIDIA Riva ASR recognition configuration.""" # Create base configuration config = riva.client.RecognitionConfig( - language_code=self._language, # Now using the string, not a tuple + language_code=self._get_language_code(), max_alternatives=1, - profanity_filter=self._profanity_filter, - enable_automatic_punctuation=self._automatic_punctuation, - verbatim_transcripts=self._verbatim_transcripts, + profanity_filter=self._settings.profanity_filter, + enable_automatic_punctuation=self._settings.automatic_punctuation, + verbatim_transcripts=self._settings.verbatim_transcripts, ) # Add word boosting if specified - if self._boosted_lm_words: + if self._settings.boosted_lm_words: riva.client.add_word_boosting_to_config( - config, self._boosted_lm_words, self._boosted_lm_score + config, self._settings.boosted_lm_words, self._settings.boosted_lm_score ) # Add voice activity detection parameters @@ -540,22 +568,6 @@ class NvidiaSegmentedSTTService(SegmentedSTTService): """ return True - async def set_model(self, model: str): - """Set the ASR model for transcription. - - Args: - model: Model name to set. - - Note: - Model cannot be changed after initialization. Use model_function_map - parameter in constructor instead. - """ - logger.warning(f"Cannot set model after initialization. Set model and function id like so:") - example = {"function_id": "", "model_name": ""} - logger.warning( - f"{self.__class__.__name__}(api_key=, model_function_map={example})" - ) - async def start(self, frame: StartFrame): """Initialize the service when the pipeline starts. @@ -565,22 +577,23 @@ class NvidiaSegmentedSTTService(SegmentedSTTService): await super().start(frame) self._initialize_client() self._config = self._create_recognition_config() - logger.debug(f"Initialized NvidiaSegmentedSTTService with model: {self.model_name}") + logger.debug(f"Initialized NvidiaSegmentedSTTService with model: {self._settings.model}") - async def set_language(self, language: Language): - """Set the language for the STT service. + async def _update_settings(self, update: STTSettings) -> dict[str, Any]: + """Apply a settings update and sync internal state. Args: - language: Target language for transcription. - """ - logger.info(f"Switching STT language to: [{language}]") - self._language_enum = language - self._language = self.language_to_service_language(language) or "en-US" - self._settings["language"] = language + update: A :class:`STTSettings` (or ``NvidiaSegmentedSTTSettings``) delta. - # Update configuration with new language - if self._config: - self._config.language_code = self._language + Returns: + Dict mapping changed field names to their previous values. + """ + changed = await super()._update_settings(update) + + if changed: + self._config = self._create_recognition_config() + + return changed @traced_stt async def _handle_transcription( @@ -633,11 +646,11 @@ class NvidiaSegmentedSTTService(SegmentedSTTService): text, self._user_id, time_now_iso8601(), - self._language_enum, + self._settings.language, ) transcription_found = True - await self._handle_transcription(text, True, self._language_enum) + await self._handle_transcription(text, True, self._settings.language) if not transcription_found: logger.debug(f"{self}: No transcription results found in NVIDIA Riva response") diff --git a/src/pipecat/services/nvidia/tts.py b/src/pipecat/services/nvidia/tts.py index 6bac54e3a..ade5da63d 100644 --- a/src/pipecat/services/nvidia/tts.py +++ b/src/pipecat/services/nvidia/tts.py @@ -12,7 +12,8 @@ gRPC API for high-quality speech synthesis. import asyncio import os -from typing import AsyncGenerator, AsyncIterator, Generator, Mapping, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, AsyncIterator, Generator, Mapping, Optional from pipecat.utils.tracing.service_decorators import traced_tts @@ -30,6 +31,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven from pipecat.services.tts_service import TTSService from pipecat.transcriptions.language import Language @@ -42,6 +44,17 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class NvidiaTTSSettings(TTSSettings): + """Settings for NVIDIA Riva TTS service. + + Parameters: + quality: Audio quality setting (0-100). + """ + + quality: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class NvidiaTTSService(TTSService): """NVIDIA Riva text-to-speech service. @@ -50,6 +63,8 @@ class NvidiaTTSService(TTSService): configurable quality settings. """ + _settings: NvidiaTTSSettings + class InputParams(BaseModel): """Input parameters for Riva TTS configuration. @@ -94,30 +109,58 @@ class NvidiaTTSService(TTSService): self._server = server self._api_key = api_key - self._voice_id = voice_id - self._language_code = params.language - self._quality = params.quality self._function_id = model_function_map.get("function_id") self._use_ssl = use_ssl - self.set_model_name(model_function_map.get("model_name")) - self.set_voice(voice_id) + self._settings = NvidiaTTSSettings( + model=model_function_map.get("model_name"), + voice=voice_id, + language=params.language, + quality=params.quality, + ) + self._sync_model_name_to_metrics() self._service = None self._config = None async def set_model(self, model: str): - """Attempt to set the TTS model. + """Set the TTS model. - Note: Model cannot be changed after initialization for Riva service. + .. deprecated:: 0.0.103 + Model cannot be changed after initialization for NVIDIA Riva TTS. + Set model and function id in the constructor instead, e.g.:: + + NvidiaTTSService( + api_key=..., + model_function_map={"function_id": "", "model_name": ""}, + ) Args: - model: The model name to set (operation not supported). + model: The model name to set. """ - logger.warning(f"Cannot set model after initialization. Set model and function id like so:") - example = {"function_id": "", "model_name": ""} - logger.warning( - f"{self.__class__.__name__}(api_key=, model_function_map={example})" - ) + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "'set_model' is deprecated. Model cannot be changed after initialization" + " for NVIDIA Riva TTS. Set model and function id in the constructor" + " instead, e.g.: NvidiaTTSService(api_key=..., model_function_map=" + "{'function_id': '', 'model_name': ''})", + DeprecationWarning, + stacklevel=2, + ) + + async def _update_settings(self, update: NvidiaTTSSettings) -> dict[str, Any]: + """Apply a settings update. + + Settings are stored but not applied to the active connection. + """ + changed = await super()._update_settings(update) + if not changed: + return changed + # TODO: reconnect gRPC client to apply changed settings. + self._warn_unhandled_updated_settings(changed) + return changed def _initialize_client(self): if self._service is not None: @@ -150,7 +193,7 @@ class NvidiaTTSService(TTSService): await super().start(frame) self._initialize_client() self._config = self._create_synthesis_config() - logger.debug(f"Initialized NvidiaTTSService with model: {self.model_name}") + logger.debug(f"Initialized NvidiaTTSService with model: {self._settings.model}") @traced_tts async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]: @@ -167,11 +210,11 @@ class NvidiaTTSService(TTSService): def read_audio_responses() -> Generator[rtts.SynthesizeSpeechResponse, None, None]: responses = self._service.synthesize_online( text, - self._voice_id, - self._language_code, + self._settings.voice, + self._settings.language, sample_rate_hz=self.sample_rate, zero_shot_audio_prompt_file=None, - zero_shot_quality=self._quality, + zero_shot_quality=self._settings.quality, custom_dictionary={}, ) return responses diff --git a/src/pipecat/services/openai/base_llm.py b/src/pipecat/services/openai/base_llm.py index ebe9eda91..960ebd6f7 100644 --- a/src/pipecat/services/openai/base_llm.py +++ b/src/pipecat/services/openai/base_llm.py @@ -10,7 +10,8 @@ import asyncio import base64 import json from contextlib import asynccontextmanager -from typing import Any, Dict, List, Mapping, Optional +from dataclasses import dataclass, field +from typing import Any, ClassVar, Dict, List, Mapping, Optional import httpx from loguru import logger @@ -32,7 +33,6 @@ from pipecat.frames.frames import ( LLMFullResponseStartFrame, LLMMessagesFrame, LLMTextFrame, - LLMUpdateSettingsFrame, ) from pipecat.metrics.metrics import LLMTokenUsage from pipecat.processors.aggregators.llm_context import LLMContext @@ -42,9 +42,24 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import FunctionCallFromLLM, LLMService +from pipecat.services.settings import NOT_GIVEN as _NOT_GIVEN +from pipecat.services.settings import LLMSettings, _NotGiven from pipecat.utils.tracing.service_decorators import traced_llm +@dataclass +class OpenAILLMSettings(LLMSettings): + """Settings for OpenAI-compatible LLM services. + + Parameters: + max_completion_tokens: Maximum completion tokens to generate. + service_tier: Service tier to use (e.g., "auto", "flex", "priority"). + """ + + max_completion_tokens: int | _NotGiven = field(default_factory=lambda: _NOT_GIVEN) + service_tier: str | _NotGiven = field(default_factory=lambda: _NOT_GIVEN) + + class BaseOpenAILLMService(LLMService): """Base class for all services that use the AsyncOpenAI client. @@ -55,6 +70,8 @@ class BaseOpenAILLMService(LLMService): configurations. """ + _settings: OpenAILLMSettings + class InputParams(BaseModel): """Input parameters for OpenAI model configuration. @@ -120,20 +137,21 @@ class BaseOpenAILLMService(LLMService): params = params or BaseOpenAILLMService.InputParams() - self._settings = { - "frequency_penalty": params.frequency_penalty, - "presence_penalty": params.presence_penalty, - "seed": params.seed, - "temperature": params.temperature, - "top_p": params.top_p, - "max_tokens": params.max_tokens, - "max_completion_tokens": params.max_completion_tokens, - "service_tier": params.service_tier, - "extra": params.extra if isinstance(params.extra, dict) else {}, - } + self._settings = OpenAILLMSettings( + model=model, + frequency_penalty=params.frequency_penalty, + presence_penalty=params.presence_penalty, + seed=params.seed, + temperature=params.temperature, + top_p=params.top_p, + max_tokens=params.max_tokens, + max_completion_tokens=params.max_completion_tokens, + service_tier=params.service_tier, + extra=params.extra if isinstance(params.extra, dict) else {}, + ) self._retry_timeout_secs = retry_timeout_secs self._retry_on_timeout = retry_on_timeout - self.set_model_name(model) + self._sync_model_name_to_metrics() self._full_model_name: str = "" self._client = self.create_client( api_key=api_key, @@ -247,23 +265,23 @@ class BaseOpenAILLMService(LLMService): Dictionary of parameters for the chat completion request. """ params = { - "model": self.model_name, + "model": self._settings.model, "stream": True, "stream_options": {"include_usage": True}, - "frequency_penalty": self._settings["frequency_penalty"], - "presence_penalty": self._settings["presence_penalty"], - "seed": self._settings["seed"], - "temperature": self._settings["temperature"], - "top_p": self._settings["top_p"], - "max_tokens": self._settings["max_tokens"], - "max_completion_tokens": self._settings["max_completion_tokens"], - "service_tier": self._settings["service_tier"], + "frequency_penalty": self._settings.frequency_penalty, + "presence_penalty": self._settings.presence_penalty, + "seed": self._settings.seed, + "temperature": self._settings.temperature, + "top_p": self._settings.top_p, + "max_tokens": self._settings.max_tokens, + "max_completion_tokens": self._settings.max_completion_tokens, + "service_tier": self._settings.service_tier, } # Messages, tools, tool_choice params.update(params_from_context) - params.update(self._settings["extra"]) + params.update(self._settings.extra) return params async def run_inference( @@ -517,8 +535,6 @@ class BaseOpenAILLMService(LLMService): # NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal # LLMContext with it context = OpenAILLMContext.from_messages(frame.messages) - elif isinstance(frame, LLMUpdateSettingsFrame): - await self._update_settings(frame.settings) else: await self.push_frame(frame, direction) diff --git a/src/pipecat/services/openai/image.py b/src/pipecat/services/openai/image.py index d6ca51ae7..36efc5987 100644 --- a/src/pipecat/services/openai/image.py +++ b/src/pipecat/services/openai/image.py @@ -53,7 +53,8 @@ class OpenAIImageGenService(ImageGenService): model: DALL-E model to use for generation. Defaults to "dall-e-3". """ super().__init__() - self.set_model_name(model) + self._settings.model = model + self._sync_model_name_to_metrics() self._image_size = image_size self._client = AsyncOpenAI(api_key=api_key, base_url=base_url) self._aiohttp_session = aiohttp_session @@ -70,7 +71,7 @@ class OpenAIImageGenService(ImageGenService): logger.debug(f"Generating image from prompt: {prompt}") image = await self._client.images.generate( - prompt=prompt, model=self.model_name, n=1, size=self._image_size + prompt=prompt, model=self._settings.model, n=1, size=self._image_size ) image_url = image.data[0].url diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index cf249408c..d765fea75 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -10,8 +10,8 @@ import base64 import io import json import time -from dataclasses import dataclass -from typing import Optional +from dataclasses import dataclass, field +from typing import Any, Optional from loguru import logger from PIL import Image @@ -59,6 +59,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import FunctionCallFromLLM, LLMService +from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven from pipecat.transcriptions.language import Language from pipecat.utils.time import time_now_iso8601 from pipecat.utils.tracing.service_decorators import traced_openai_realtime, traced_stt @@ -90,6 +91,19 @@ class CurrentAudioResponse: total_size: int = 0 +@dataclass +class OpenAIRealtimeLLMSettings(LLMSettings): + """Settings for OpenAI Realtime LLM services. + + Parameters: + session_properties: OpenAI Realtime session configuration. + """ + + session_properties: events.SessionProperties | _NotGiven = field( + default_factory=lambda: NOT_GIVEN + ) + + class OpenAIRealtimeLLMService(LLMService): """OpenAI Realtime LLM service providing real-time audio and text communication. @@ -98,6 +112,8 @@ class OpenAIRealtimeLLMService(LLMService): management, and real-time transcription. """ + _settings: OpenAIRealtimeLLMSettings + # Overriding the default adapter to use the OpenAIRealtimeLLMAdapter one. adapter_class = OpenAIRealtimeLLMAdapter @@ -159,12 +175,12 @@ class OpenAIRealtimeLLMService(LLMService): self.api_key = api_key self.base_url = full_url - self.set_model_name(model) - # Initialize session_properties - self._session_properties: events.SessionProperties = ( - session_properties or events.SessionProperties() + self._settings = OpenAIRealtimeLLMSettings( + model=model, + session_properties=session_properties or events.SessionProperties(), ) + self._sync_model_name_to_metrics() self._audio_input_paused = start_audio_paused self._video_input_paused = start_video_paused self._video_frame_detail = video_frame_detail @@ -227,12 +243,12 @@ class OpenAIRealtimeLLMService(LLMService): def _is_modality_enabled(self, modality: str) -> bool: """Check if a specific modality is enabled, "text" or "audio".""" - modalities = self._session_properties.output_modalities or ["audio", "text"] + modalities = self._settings.session_properties.output_modalities or ["audio", "text"] return modality in modalities def _get_enabled_modalities(self) -> list[str]: """Get the list of enabled modalities.""" - modalities = self._session_properties.output_modalities or ["audio", "text"] + modalities = self._settings.session_properties.output_modalities or ["audio", "text"] # API only supports single modality responses: either ["text"] or ["audio"] if "audio" in modalities: return ["audio"] @@ -305,9 +321,9 @@ class OpenAIRealtimeLLMService(LLMService): # None and False are different. Check for False. None means we're using OpenAI's # built-in turn detection defaults. turn_detection_disabled = ( - self._session_properties.audio - and self._session_properties.audio.input - and self._session_properties.audio.input.turn_detection is False + self._settings.session_properties.audio + and self._settings.session_properties.audio.input + and self._settings.session_properties.audio.input.turn_detection is False ) if turn_detection_disabled: await self.send_client_event(events.InputAudioBufferClearEvent()) @@ -327,9 +343,9 @@ class OpenAIRealtimeLLMService(LLMService): # None and False are different. Check for False. None means we're using OpenAI's # built-in turn detection defaults. turn_detection_disabled = ( - self._session_properties.audio - and self._session_properties.audio.input - and self._session_properties.audio.input.turn_detection is False + self._settings.session_properties.audio + and self._settings.session_properties.audio.input + and self._settings.session_properties.audio.input.turn_detection is False ) if turn_detection_disabled: await self.send_client_event(events.InputAudioBufferCommitEvent()) @@ -397,6 +413,16 @@ class OpenAIRealtimeLLMService(LLMService): frame: The frame to process. direction: The direction of frame flow in the pipeline. """ + # Backward-compatible dict path: frame.settings contains SessionProperties + # fields, not our Settings fields, so we construct SessionProperties + # directly. The frame.update path falls through to super, which calls + # _update_settings → our override handles the rest. + if isinstance(frame, LLMUpdateSettingsFrame) and frame.update is None: + self._settings.session_properties = events.SessionProperties(**frame.settings) + await self._send_session_update() + await self.push_frame(frame, direction) + return + await super().process_frame(frame, direction) if isinstance(frame, TranscriptionFrame): @@ -424,11 +450,8 @@ class OpenAIRealtimeLLMService(LLMService): await self._handle_bot_stopped_speaking() elif isinstance(frame, LLMMessagesAppendFrame): await self._handle_messages_append(frame) - elif isinstance(frame, LLMUpdateSettingsFrame): - self._session_properties = events.SessionProperties(**frame.settings) - await self._update_settings() elif isinstance(frame, LLMSetToolsFrame): - await self._update_settings() + await self._send_session_update() await self.push_frame(frame, direction) @@ -513,8 +536,16 @@ class OpenAIRealtimeLLMService(LLMService): # treat a send-side error as fatal. await self.push_error(error_msg=f"Error sending client event: {e}", exception=e) - async def _update_settings(self): - settings = self._session_properties + async def _update_settings(self, update): + """Apply a settings update, sending a session update if needed.""" + changed = await super()._update_settings(update) + if "session_properties" in changed: + await self._send_session_update() + self._warn_unhandled_updated_settings(changed.keys() - {"session_properties"}) + return changed + + async def _send_session_update(self): + settings = self._settings.session_properties adapter: OpenAIRealtimeLLMAdapter = self.get_llm_adapter() if self._context: @@ -585,7 +616,7 @@ class OpenAIRealtimeLLMService(LLMService): async def _handle_evt_session_created(self, evt): # session.created is received right after connecting. Send a message # to configure the session properties. - await self._update_settings() + await self._send_session_update() async def _handle_evt_session_updated(self, evt): # If this is our first context frame, run the LLM @@ -868,7 +899,7 @@ class OpenAIRealtimeLLMService(LLMService): await self.send_client_event(evt) # Send new settings if needed - await self._update_settings() + await self._send_session_update() # We're done configuring the LLM for this session self._llm_needs_conversation_setup = False diff --git a/src/pipecat/services/openai/stt.py b/src/pipecat/services/openai/stt.py index 4dd16be6e..13a37a2b1 100644 --- a/src/pipecat/services/openai/stt.py +++ b/src/pipecat/services/openai/stt.py @@ -16,7 +16,8 @@ Provides two STT services: import base64 import json -from typing import AsyncGenerator, Literal, Optional, Union +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, Literal, Optional, Union from loguru import logger @@ -34,6 +35,7 @@ from pipecat.frames.frames import ( VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven, is_given from pipecat.services.stt_latency import OPENAI_REALTIME_TTFS_P99, OPENAI_TTFS_P99 from pipecat.services.stt_service import WebsocketSTTService from pipecat.services.whisper.base_stt import BaseWhisperSTTService, Transcription @@ -98,24 +100,24 @@ class OpenAISTTService(BaseWhisperSTTService): # Build kwargs dict with only set parameters kwargs = { "file": ("audio.wav", audio, "audio/wav"), - "model": self.model_name, - "language": self._language, + "model": self._settings.model, + "language": self._settings.language, } if self._include_prob_metrics: # GPT-4o-transcribe models only support logprobs (not verbose_json) - if self.model_name in ("gpt-4o-transcribe", "gpt-4o-mini-transcribe"): + if self._settings.model in ("gpt-4o-transcribe", "gpt-4o-mini-transcribe"): kwargs["response_format"] = "json" kwargs["include"] = ["logprobs"] else: # Whisper models support verbose_json kwargs["response_format"] = "verbose_json" - if self._prompt is not None: - kwargs["prompt"] = self._prompt + if self._settings.prompt is not None: + kwargs["prompt"] = self._settings.prompt - if self._temperature is not None: - kwargs["temperature"] = self._temperature + if self._settings.temperature is not None: + kwargs["temperature"] = self._settings.temperature return await self._client.audio.transcriptions.create(**kwargs) @@ -123,6 +125,17 @@ class OpenAISTTService(BaseWhisperSTTService): _OPENAI_SAMPLE_RATE = 24000 +@dataclass +class OpenAIRealtimeSTTSettings(STTSettings): + """Settings for the OpenAI Realtime STT service. + + Parameters: + prompt: Optional prompt text to guide transcription style. + """ + + prompt: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class OpenAIRealtimeSTTService(WebsocketSTTService): """OpenAI Realtime Speech-to-Text service using WebSocket transcription sessions. @@ -156,6 +169,8 @@ class OpenAIRealtimeSTTService(WebsocketSTTService): ) """ + _settings: OpenAIRealtimeSTTSettings + def __init__( self, *, @@ -211,14 +226,19 @@ class OpenAIRealtimeSTTService(WebsocketSTTService): self._api_key = api_key self._base_url = base_url - self.set_model_name(model) - self._language_code = self._language_to_code(language) if language else None self._prompt = prompt self._turn_detection = turn_detection self._noise_reduction = noise_reduction self._should_interrupt = should_interrupt + self._settings = OpenAIRealtimeSTTSettings( + model=model, + language=language, + prompt=prompt, + ) + self._sync_model_name_to_metrics() + self._receive_task = None self._session_ready = False self._resampler = create_stream_resampler() @@ -248,19 +268,31 @@ class OpenAIRealtimeSTTService(WebsocketSTTService): """ return True - async def set_language(self, language: Language): - """Set the language for speech recognition. + async def _update_settings(self, update: STTSettings) -> dict[str, Any]: + """Apply a settings update and send session update if needed. - If the session is already active, sends an updated configuration - to the server. + Keeps ``_language_code`` and ``_prompt`` in sync with settings + and sends a ``session.update`` to the server when the session is active. Args: - language: The language to use for speech recognition. + update: A :class:`STTSettings` (or ``OpenAIRealtimeSTTSettings``) delta. + + Returns: + Dict mapping changed field names to their previous values. """ - self._language_code = self._language_to_code(language) + changed = await super()._update_settings(update) + + if not changed: + return changed + + if "prompt" in changed and isinstance(self._settings, OpenAIRealtimeSTTSettings): + self._prompt = self._settings.prompt + if self._session_ready: await self._send_session_update() + return changed + async def start(self, frame: StartFrame): """Start the service and establish WebSocket connection. @@ -405,10 +437,13 @@ class OpenAIRealtimeSTTService(WebsocketSTTService): async def _send_session_update(self): """Send ``session.update`` to configure the transcription session.""" - transcription: dict = {"model": self.model_name} + transcription: dict = {"model": self._settings.model} - if self._language_code: - transcription["language"] = self._language_code + language_code = ( + self._language_to_code(self._settings.language) if self._settings.language else None + ) + if language_code: + transcription["language"] = language_code if self._prompt: transcription["prompt"] = self._prompt diff --git a/src/pipecat/services/openai/tts.py b/src/pipecat/services/openai/tts.py index f59f0b31b..2693bcc27 100644 --- a/src/pipecat/services/openai/tts.py +++ b/src/pipecat/services/openai/tts.py @@ -10,6 +10,7 @@ This module provides integration with OpenAI's text-to-speech API for generating high-quality synthetic speech from text input. """ +from dataclasses import dataclass, field from typing import AsyncGenerator, Dict, Literal, Optional from loguru import logger @@ -24,6 +25,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven from pipecat.services.tts_service import TTSService from pipecat.utils.tracing.service_decorators import traced_tts @@ -60,6 +62,19 @@ VALID_VOICES: Dict[str, ValidVoice] = { } +@dataclass +class OpenAITTSSettings(TTSSettings): + """Settings for OpenAI TTS service. + + Parameters: + instructions: Instructions to guide voice synthesis behavior. + speed: Voice speed control (0.25 to 4.0, default 1.0). + """ + + instructions: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + speed: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class OpenAITTSService(TTSService): """OpenAI Text-to-Speech service that generates audio from text. @@ -68,6 +83,8 @@ class OpenAITTSService(TTSService): speech synthesis with streaming audio output. """ + _settings: OpenAITTSSettings + OPENAI_SAMPLE_RATE = 24000 # OpenAI TTS always outputs at 24kHz class InputParams(BaseModel): @@ -117,8 +134,6 @@ class OpenAITTSService(TTSService): ) super().__init__(sample_rate=sample_rate, **kwargs) - self.set_model_name(model) - self.set_voice(voice) self._client = AsyncOpenAI(api_key=api_key, base_url=base_url) if instructions or speed: @@ -132,10 +147,13 @@ class OpenAITTSService(TTSService): stacklevel=2, ) - self._settings = { - "instructions": params.instructions if params else instructions, - "speed": params.speed if params else speed, - } + self._settings = OpenAITTSSettings( + model=model, + voice=voice, + instructions=params.instructions if params else instructions, + speed=params.speed if params else speed, + ) + self._sync_model_name_to_metrics() def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -145,15 +163,6 @@ class OpenAITTSService(TTSService): """ return True - async def set_model(self, model: str): - """Set the TTS model to use. - - Args: - model: The model name to use for text-to-speech synthesis. - """ - logger.info(f"Switching TTS model to: [{model}]") - self.set_model_name(model) - async def start(self, frame: StartFrame): """Start the OpenAI TTS service. @@ -185,16 +194,16 @@ class OpenAITTSService(TTSService): # Setup API parameters create_params = { "input": text, - "model": self.model_name, - "voice": VALID_VOICES[self._voice_id], + "model": self._settings.model, + "voice": VALID_VOICES[self._settings.voice], "response_format": "pcm", } - if self._settings["instructions"]: - create_params["instructions"] = self._settings["instructions"] + if self._settings.instructions: + create_params["instructions"] = self._settings.instructions - if self._settings["speed"]: - create_params["speed"] = self._settings["speed"] + if self._settings.speed: + create_params["speed"] = self._settings.speed async with self._client.audio.speech.with_streaming_response.create( **create_params diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 1199d8556..efac34223 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -10,7 +10,7 @@ import base64 import json import time import warnings -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Optional from loguru import logger @@ -54,6 +54,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import FunctionCallFromLLM, LLMService from pipecat.services.openai.llm import OpenAIContextAggregatorPair +from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven from pipecat.transcriptions.language import Language from pipecat.utils.time import time_now_iso8601 from pipecat.utils.tracing.service_decorators import traced_openai_realtime, traced_stt @@ -91,6 +92,19 @@ class CurrentAudioResponse: total_size: int = 0 +@dataclass +class OpenAIRealtimeBetaLLMSettings(LLMSettings): + """Settings for OpenAI Realtime Beta LLM services. + + Parameters: + session_properties: OpenAI Realtime session configuration. + """ + + session_properties: events.SessionProperties | _NotGiven = field( + default_factory=lambda: NOT_GIVEN + ) + + class OpenAIRealtimeBetaLLMService(LLMService): """OpenAI Realtime Beta LLM service providing real-time audio and text communication. @@ -103,6 +117,8 @@ class OpenAIRealtimeBetaLLMService(LLMService): management, and real-time transcription. """ + _settings: OpenAIRealtimeBetaLLMSettings + # Overriding the default adapter to use the OpenAIRealtimeLLMAdapter one. adapter_class = OpenAIRealtimeLLMAdapter @@ -144,11 +160,12 @@ class OpenAIRealtimeBetaLLMService(LLMService): self.api_key = api_key self.base_url = full_url - self.set_model_name(model) - self._session_properties: events.SessionProperties = ( - session_properties or events.SessionProperties() + self._settings = OpenAIRealtimeBetaLLMSettings( + model=model, + session_properties=session_properties or events.SessionProperties(), ) + self._sync_model_name_to_metrics() self._audio_input_paused = start_audio_paused self._send_transcription_frames = send_transcription_frames self._websocket = None @@ -187,12 +204,12 @@ class OpenAIRealtimeBetaLLMService(LLMService): def _is_modality_enabled(self, modality: str) -> bool: """Check if a specific modality is enabled, "text" or "audio".""" - modalities = self._session_properties.modalities or ["audio", "text"] + modalities = self._settings.session_properties.modalities or ["audio", "text"] return modality in modalities def _get_enabled_modalities(self) -> list[str]: """Get the list of enabled modalities.""" - return self._session_properties.modalities or ["audio", "text"] + return self._settings.session_properties.modalities or ["audio", "text"] async def retrieve_conversation_item(self, item_id: str): """Retrieve a conversation item by ID from the server. @@ -259,7 +276,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): async def _handle_interruption(self): # None and False are different. Check for False. None means we're using OpenAI's # built-in turn detection defaults. - if self._session_properties.turn_detection is False: + if self._settings.session_properties.turn_detection is False: await self.send_client_event(events.InputAudioBufferClearEvent()) await self.send_client_event(events.ResponseCancelEvent()) await self._truncate_current_audio_response() @@ -276,7 +293,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): async def _handle_user_stopped_speaking(self, frame): # None and False are different. Check for False. None means we're using OpenAI's # built-in turn detection defaults. - if self._session_properties.turn_detection is False: + if self._settings.session_properties.turn_detection is False: await self.send_client_event(events.InputAudioBufferCommitEvent()) await self.send_client_event(events.ResponseCreateEvent()) @@ -342,6 +359,16 @@ class OpenAIRealtimeBetaLLMService(LLMService): frame: The frame to process. direction: The direction of frame flow in the pipeline. """ + # Backward-compatible dict path: frame.settings contains SessionProperties + # fields, not our Settings fields, so we construct SessionProperties + # directly. The frame.update path falls through to super, which calls + # _update_settings → our override handles the rest. + if isinstance(frame, LLMUpdateSettingsFrame) and frame.update is None: + self._settings.session_properties = events.SessionProperties(**frame.settings) + await self._send_session_update() + await self.push_frame(frame, direction) + return + await super().process_frame(frame, direction) if isinstance(frame, TranscriptionFrame): @@ -377,11 +404,8 @@ class OpenAIRealtimeBetaLLMService(LLMService): await self._handle_messages_append(frame) elif isinstance(frame, RealtimeMessagesUpdateFrame): self._context = frame.context - elif isinstance(frame, LLMUpdateSettingsFrame): - self._session_properties = events.SessionProperties(**frame.settings) - await self._update_settings() elif isinstance(frame, LLMSetToolsFrame): - await self._update_settings() + await self._send_session_update() elif isinstance(frame, RealtimeFunctionCallResultFrame): await self._handle_function_call_result(frame.result_frame) @@ -456,8 +480,15 @@ class OpenAIRealtimeBetaLLMService(LLMService): # treat a send-side error as fatal. await self.push_error(error_msg=f"Error sending client event: {e}", exception=e) - async def _update_settings(self): - settings = self._session_properties + async def _update_settings(self, update): + """Apply a settings update, sending a session update if needed.""" + changed = await super()._update_settings(update) + if "session_properties" in changed: + await self._send_session_update() + return changed + + async def _send_session_update(self): + settings = self._settings.session_properties # tools given in the context override the tools in the session properties if self._context and self._context.tools: settings.tools = self._context.tools @@ -511,7 +542,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): async def _handle_evt_session_created(self, evt): # session.created is received right after connecting. Send a message # to configure the session properties. - await self._update_settings() + await self._send_session_update() async def _handle_evt_session_updated(self, evt): # If this is our first context frame, run the LLM @@ -750,7 +781,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): self._context.llm_needs_initial_messages = False if self._context.llm_needs_settings_update: - await self._update_settings() + await self._send_session_update() self._context.llm_needs_settings_update = False logger.debug(f"Creating response: {self._context.get_messages_for_logging()}") diff --git a/src/pipecat/services/openrouter/llm.py b/src/pipecat/services/openrouter/llm.py index a86b18573..c33fda2fc 100644 --- a/src/pipecat/services/openrouter/llm.py +++ b/src/pipecat/services/openrouter/llm.py @@ -72,8 +72,7 @@ class OpenRouterLLMService(OpenAILLMService): Transformed parameters ready for the API call. """ params = super().build_chat_completion_params(params_from_context) - model = getattr(self, "model_name", getattr(self, "model", "")).lower() - if "gemini" in model: + if "gemini" in self._settings.model.lower(): messages = params.get("messages", []) if not messages: return params diff --git a/src/pipecat/services/perplexity/llm.py b/src/pipecat/services/perplexity/llm.py index 4ea23aa82..04f25621d 100644 --- a/src/pipecat/services/perplexity/llm.py +++ b/src/pipecat/services/perplexity/llm.py @@ -66,22 +66,22 @@ class PerplexityLLMService(OpenAILLMService): Dictionary of parameters for the chat completion request. """ params = { - "model": self.model_name, + "model": self._settings.model, "stream": True, "messages": params_from_context["messages"], } # Add OpenAI-compatible parameters if they're set - if self._settings["frequency_penalty"] is not NOT_GIVEN: - params["frequency_penalty"] = self._settings["frequency_penalty"] - if self._settings["presence_penalty"] is not NOT_GIVEN: - params["presence_penalty"] = self._settings["presence_penalty"] - if self._settings["temperature"] is not NOT_GIVEN: - params["temperature"] = self._settings["temperature"] - if self._settings["top_p"] is not NOT_GIVEN: - params["top_p"] = self._settings["top_p"] - if self._settings["max_tokens"] is not NOT_GIVEN: - params["max_tokens"] = self._settings["max_tokens"] + if self._settings.frequency_penalty is not NOT_GIVEN: + params["frequency_penalty"] = self._settings.frequency_penalty + if self._settings.presence_penalty is not NOT_GIVEN: + params["presence_penalty"] = self._settings.presence_penalty + if self._settings.temperature is not NOT_GIVEN: + params["temperature"] = self._settings.temperature + if self._settings.top_p is not NOT_GIVEN: + params["top_p"] = self._settings.top_p + if self._settings.max_tokens is not NOT_GIVEN: + params["max_tokens"] = self._settings.max_tokens return params diff --git a/src/pipecat/services/piper/tts.py b/src/pipecat/services/piper/tts.py index a1a038826..0b43d96d2 100644 --- a/src/pipecat/services/piper/tts.py +++ b/src/pipecat/services/piper/tts.py @@ -7,8 +7,9 @@ """Piper TTS service implementation.""" import asyncio +from dataclasses import dataclass from pathlib import Path -from typing import AsyncGenerator, AsyncIterator, Optional +from typing import Any, AsyncGenerator, AsyncIterator, Optional import aiohttp from loguru import logger @@ -19,6 +20,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import TTSSettings from pipecat.services.tts_service import TTSService from pipecat.utils.tracing.service_decorators import traced_tts @@ -31,6 +33,13 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class PiperTTSSettings(TTSSettings): + """Settings for Piper TTS service.""" + + pass + + class PiperTTSService(TTSService): """Piper TTS service implementation. @@ -39,6 +48,8 @@ class PiperTTSService(TTSService): match the configured sample rate. """ + _settings: PiperTTSSettings + def __init__( self, *, @@ -60,7 +71,7 @@ class PiperTTSService(TTSService): """ super().__init__(**kwargs) - self._voice_id = voice_id + self._settings = PiperTTSSettings(voice=voice_id) download_dir = download_dir or Path.cwd() @@ -85,6 +96,18 @@ class PiperTTSService(TTSService): """ return True + async def _update_settings(self, update: PiperTTSSettings) -> dict[str, Any]: + """Apply a settings update. + + Settings are stored but not applied to the active connection. + """ + changed = await super()._update_settings(update) + if not changed: + return changed + # TODO: voice changes would require re-downloading and loading the model. + self._warn_unhandled_updated_settings(changed) + return changed + @traced_tts async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]: """Generate speech from text using Piper. @@ -143,6 +166,13 @@ class PiperTTSService(TTSService): # $ uv pip install "piper-tts[http]" # $ uv run python -m piper.http_server -m en_US-ryan-high # +@dataclass +class PiperHttpTTSSettings(TTSSettings): + """Settings for Piper HTTP TTS service.""" + + pass + + class PiperHttpTTSService(TTSService): """Piper HTTP TTS service implementation. @@ -151,6 +181,8 @@ class PiperHttpTTSService(TTSService): rates and automatic WAV header removal. """ + _settings: PiperHttpTTSSettings + def __init__( self, *, @@ -175,7 +207,7 @@ class PiperHttpTTSService(TTSService): self._base_url = base_url self._session = aiohttp_session - self._model_id = voice_id + self._settings = PiperHttpTTSSettings(voice=voice_id) def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -205,7 +237,7 @@ class PiperHttpTTSService(TTSService): data = { "text": text, - "voice": self._model_id, + "voice": self._settings.voice, } async with self._session.post(self._base_url, json=data, headers=headers) as response: diff --git a/src/pipecat/services/playht/tts.py b/src/pipecat/services/playht/tts.py index 287463186..ff34725f9 100644 --- a/src/pipecat/services/playht/tts.py +++ b/src/pipecat/services/playht/tts.py @@ -15,7 +15,8 @@ import json import struct import uuid import warnings -from typing import AsyncGenerator, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, Optional import aiohttp from loguru import logger @@ -33,6 +34,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven, is_given from pipecat.services.tts_service import InterruptibleTTSService, TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -98,6 +100,25 @@ def language_to_playht_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +@dataclass +class PlayHTTTSSettings(TTSSettings): + """Settings for PlayHT TTS services. + + Parameters: + output_format: Audio output format. + voice_engine: Voice engine to use. + speed: Speech speed multiplier. Defaults to 1.0. + seed: Random seed for voice consistency. + playht_sample_rate: Audio sample rate sent to the API. + """ + + output_format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + voice_engine: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + seed: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + playht_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class PlayHTTTSService(InterruptibleTTSService): """PlayHT WebSocket-based text-to-speech service. @@ -111,6 +132,8 @@ class PlayHTTTSService(InterruptibleTTSService): language settings. """ + _settings: PlayHTTTSSettings + class InputParams(BaseModel): """Input parameters for PlayHT TTS configuration. @@ -171,17 +194,18 @@ class PlayHTTTSService(InterruptibleTTSService): self._receive_task = None self._context_id = None - self._settings = { - "language": self.language_to_service_language(params.language) + self._settings = PlayHTTTSSettings( + model=voice_engine, + voice=voice_url, + language=self.language_to_service_language(params.language) if params.language else "english", - "output_format": output_format, - "voice_engine": voice_engine, - "speed": params.speed, - "seed": params.seed, - } - self.set_model_name(voice_engine) - self.set_voice(voice_url) + output_format=output_format, + voice_engine=voice_engine, + speed=params.speed, + seed=params.seed, + ) + self._sync_model_name_to_metrics() def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -191,6 +215,25 @@ class PlayHTTTSService(InterruptibleTTSService): """ return True + async def _update_settings(self, update: TTSSettings) -> dict[str, Any]: + """Apply a settings update. + + Settings are stored but not applied to the active connection. + """ + changed = await super()._update_settings(update) + + if not changed: + return changed + + # TODO: someday we could reconnect here to apply updated settings. + # Code might look something like the below: + # await self._disconnect() + # await self._connect() + + self._warn_unhandled_updated_settings(changed) + + return changed + def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to PlayHT service language format. @@ -305,13 +348,13 @@ class PlayHTTTSService(InterruptibleTTSService): # Handle the new response format with multiple URLs if "websocket_urls" in data: # Select URL based on voice_engine - if self._settings["voice_engine"] in data["websocket_urls"]: + if self._settings.voice_engine in data["websocket_urls"]: self._websocket_url = data["websocket_urls"][ - self._settings["voice_engine"] + self._settings.voice_engine ] else: raise ValueError( - f"Unsupported voice engine: {self._settings['voice_engine']}" + f"Unsupported voice engine: {self._settings.voice_engine}" ) else: raise ValueError("Invalid response: missing websocket_urls") @@ -396,13 +439,13 @@ class PlayHTTTSService(InterruptibleTTSService): tts_command = { "text": text, - "voice": self._voice_id, - "voice_engine": self._settings["voice_engine"], - "output_format": self._settings["output_format"], + "voice": self._settings.voice, + "voice_engine": self._settings.voice_engine, + "output_format": self._settings.output_format, "sample_rate": self.sample_rate, - "language": self._settings["language"], - "speed": self._settings["speed"], - "seed": self._settings["seed"], + "language": self._settings.language, + "speed": self._settings.speed, + "seed": self._settings.seed, "request_id": self._context_id, } @@ -436,6 +479,8 @@ class PlayHTHttpTTSService(TTSService): required and simpler integration is preferred. """ + _settings: PlayHTTTSSettings + class InputParams(BaseModel): """Input parameters for PlayHT HTTP TTS configuration. @@ -514,17 +559,18 @@ class PlayHTHttpTTSService(TTSService): # Extract the base engine name voice_engine = voice_engine.replace("-ws", "") - self._settings = { - "language": self.language_to_service_language(params.language) + self._settings = PlayHTTTSSettings( + model=voice_engine, + voice=voice_url, + language=self.language_to_service_language(params.language) if params.language else "english", - "output_format": output_format, - "voice_engine": voice_engine, - "speed": params.speed, - "seed": params.seed, - } - self.set_model_name(voice_engine) - self.set_voice(voice_url) + output_format=output_format, + voice_engine=voice_engine, + speed=params.speed, + seed=params.seed, + ) + self._sync_model_name_to_metrics() async def start(self, frame: StartFrame): """Start the PlayHT HTTP TTS service. @@ -533,7 +579,7 @@ class PlayHTHttpTTSService(TTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["sample_rate"] = self.sample_rate + self._settings.playht_sample_rate = self.sample_rate def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -573,18 +619,18 @@ class PlayHTHttpTTSService(TTSService): # Prepare the request payload payload = { "text": text, - "voice": self._voice_id, - "voice_engine": self._settings["voice_engine"], - "output_format": self._settings["output_format"], + "voice": self._settings.voice, + "voice_engine": self._settings.voice_engine, + "output_format": self._settings.output_format, "sample_rate": self.sample_rate, - "language": self._settings["language"], + "language": self._settings.language, } # Add optional parameters if they exist - if self._settings["speed"] is not None: - payload["speed"] = self._settings["speed"] - if self._settings["seed"] is not None: - payload["seed"] = self._settings["seed"] + if self._settings.speed is not None: + payload["speed"] = self._settings.speed + if self._settings.seed is not None: + payload["seed"] = self._settings.seed headers = { "Authorization": f"Bearer {self._api_key}", diff --git a/src/pipecat/services/resembleai/tts.py b/src/pipecat/services/resembleai/tts.py index c51ccc07b..b8bb4a1da 100644 --- a/src/pipecat/services/resembleai/tts.py +++ b/src/pipecat/services/resembleai/tts.py @@ -8,7 +8,8 @@ import base64 import json -from typing import AsyncGenerator, Optional +from dataclasses import dataclass, field +from typing import AsyncGenerator, ClassVar, Dict, Optional from loguru import logger @@ -24,6 +25,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven from pipecat.services.tts_service import AudioContextWordTTSService from pipecat.utils.tracing.service_decorators import traced_tts @@ -36,6 +38,26 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class ResembleAITTSSettings(TTSSettings): + """Settings for Resemble AI TTS service. + + Parameters: + precision: PCM bit depth (PCM_32, PCM_24, PCM_16, or MULAW). + output_format: Audio format (wav or mp3). + resemble_sample_rate: Audio sample rate sent to the API. + """ + + precision: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + output_format: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + resemble_sample_rate: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + _aliases: ClassVar[Dict[str, str]] = { + "voice_id": "voice", + "sample_rate": "resemble_sample_rate", + } + + class ResembleAITTSService(AudioContextWordTTSService): """Resemble AI TTS service with WebSocket streaming and word timestamps. @@ -44,6 +66,8 @@ class ResembleAITTSService(AudioContextWordTTSService): multiple simultaneous synthesis requests with proper interruption support. """ + _settings: ResembleAITTSSettings + def __init__( self, *, @@ -73,13 +97,13 @@ class ResembleAITTSService(AudioContextWordTTSService): ) self._api_key = api_key - self._voice_id = voice_id self._url = url - self._settings = { - "precision": precision, - "output_format": output_format, - "sample_rate": sample_rate, - } + self._settings = ResembleAITTSSettings( + voice=voice_id, + precision=precision, + output_format=output_format, + resemble_sample_rate=sample_rate, + ) self._websocket = None self._request_id_counter = 0 @@ -100,8 +124,6 @@ class ResembleAITTSService(AudioContextWordTTSService): self._jitter_buffer_bytes = 44100 # ~1000ms at 22050Hz to handle 400ms+ network gaps self._playback_started: dict[str, bool] = {} # Track if we've started playback per request - self.set_voice(voice_id) - def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -120,13 +142,13 @@ class ResembleAITTSService(AudioContextWordTTSService): JSON string containing the request payload. """ msg = { - "voice_uuid": self._voice_id, + "voice_uuid": self._settings.voice, "data": text, "binary_response": False, # Use JSON frames to get timestamps "request_id": self._request_id_counter, # ResembleAI only accepts number - "output_format": self._settings["output_format"], - "sample_rate": self._settings["sample_rate"], - "precision": self._settings["precision"], + "output_format": self._settings.output_format, + "sample_rate": self._settings.resemble_sample_rate, + "precision": self._settings.precision, "no_audio_header": True, } @@ -140,7 +162,7 @@ class ResembleAITTSService(AudioContextWordTTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["sample_rate"] = self.sample_rate + self._settings.resemble_sample_rate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index c4b1c870a..6e03c2461 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -12,7 +12,8 @@ using Rime's API for streaming and batch audio synthesis. import base64 import json -from typing import Any, AsyncGenerator, Mapping, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, ClassVar, Dict, Optional import aiohttp from loguru import logger @@ -30,6 +31,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven, is_given from pipecat.services.tts_service import ( AudioContextWordTTSService, InterruptibleTTSService, @@ -68,6 +70,66 @@ def language_to_rime_language(language: Language) -> str: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +@dataclass +class RimeTTSSettings(TTSSettings): + """Settings for Rime WS JSON and HTTP TTS services. + + Parameters: + audioFormat: Audio output format. + samplingRate: Audio sample rate. + segment: Text segmentation mode ("immediate", "bySentence", "never"). + speedAlpha: Speech speed multiplier (mistv2 only). + reduceLatency: Whether to reduce latency at potential quality cost (mistv2 only). + pauseBetweenBrackets: Whether to add pauses between bracketed content (mistv2 only). + phonemizeBetweenBrackets: Whether to phonemize bracketed content (mistv2 only). + noTextNormalization: Whether to disable text normalization (mistv2 only). + saveOovs: Whether to save out-of-vocabulary words (mistv2 only). + inlineSpeedAlpha: Inline speed control markup. + repetition_penalty: Token repetition penalty (arcana only, 1.0-2.0). + temperature: Sampling temperature (arcana only, 0.0-1.0). + top_p: Cumulative probability threshold (arcana only, 0.0-1.0). + """ + + audioFormat: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + samplingRate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + segment: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + speedAlpha: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + reduceLatency: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + pauseBetweenBrackets: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + phonemizeBetweenBrackets: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + noTextNormalization: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + saveOovs: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + inlineSpeedAlpha: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + repetition_penalty: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + temperature: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + top_p: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + _aliases: ClassVar[Dict[str, str]] = {"speaker": "voice"} + + +@dataclass +class RimeNonJsonTTSSettings(TTSSettings): + """Settings for Rime non-JSON WS TTS service. + + Parameters: + audioFormat: Audio output format. + samplingRate: Audio sample rate. + segment: Text segmentation mode ("immediate", "bySentence", "never"). + repetition_penalty: Token repetition penalty (1.0-2.0). + temperature: Sampling temperature (0.0-1.0). + top_p: Cumulative probability threshold (0.0-1.0). + """ + + audioFormat: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + samplingRate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + segment: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + repetition_penalty: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + temperature: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + top_p: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + _aliases: ClassVar[Dict[str, str]] = {"speaker": "voice"} + + class RimeTTSService(AudioContextWordTTSService): """Text-to-Speech service using Rime's websocket API. @@ -76,6 +138,8 @@ class RimeTTSService(AudioContextWordTTSService): within a turn. """ + _settings: RimeTTSSettings + class InputParams(BaseModel): """Configuration parameters for Rime TTS service. @@ -156,14 +220,41 @@ class RimeTTSService(AudioContextWordTTSService): # and insert these tags for the purpose of the TTS service alone. self._text_aggregator = SkipTagsAggregator([("spell(", ")")]) - self._params = params or RimeTTSService.InputParams() + params = params or RimeTTSService.InputParams() # Store service configuration self._api_key = api_key self._url = url - self._voice_id = voice_id - self._model = model - self._settings = self._build_settings() + self._settings = RimeTTSSettings( + model=model, + voice=voice_id, + audioFormat="pcm", + samplingRate=0, # updated in start() + language=self.language_to_service_language(params.language) + if params.language + else NOT_GIVEN, + segment=params.segment if params.segment is not None else NOT_GIVEN, + # Arcana params + repetition_penalty=params.repetition_penalty + if params.repetition_penalty is not None + else NOT_GIVEN, + temperature=params.temperature if params.temperature is not None else NOT_GIVEN, + top_p=params.top_p if params.top_p is not None else NOT_GIVEN, + # Mistv2 params + speedAlpha=params.speed_alpha if params.speed_alpha is not None else NOT_GIVEN, + reduceLatency=params.reduce_latency if params.reduce_latency is not None else NOT_GIVEN, + pauseBetweenBrackets=params.pause_between_brackets + if params.pause_between_brackets is not None + else NOT_GIVEN, + phonemizeBetweenBrackets=params.phonemize_between_brackets + if params.phonemize_between_brackets is not None + else NOT_GIVEN, + noTextNormalization=params.no_text_normalization + if params.no_text_normalization is not None + else NOT_GIVEN, + saveOovs=params.save_oovs if params.save_oovs is not None else NOT_GIVEN, + ) + self._sync_model_name_to_metrics() # State tracking self._receive_task = None @@ -189,60 +280,49 @@ class RimeTTSService(AudioContextWordTTSService): """ return language_to_rime_language(language) - def _build_settings(self) -> dict: - """Build query params for the WebSocket URL based on the current model and params. + def _build_ws_params(self) -> dict[str, Any]: + """Build query params for the WebSocket URL from current settings. Returns: - Dictionary of query parameters. Only explicitly-set values are included. + Dictionary of query parameters for the WebSocket URL. + Only explicitly-set values are included. Boolean mistv2 params + are serialized with ``json.dumps()`` for the wire format. """ - settings = { - "speaker": self._voice_id, - "modelId": self._model, - "audioFormat": "pcm", - "samplingRate": self.sample_rate or 0, + params: dict[str, Any] = { + "speaker": self._settings.voice, + "modelId": self._settings.model, + "audioFormat": self._settings.audioFormat, + "samplingRate": self._settings.samplingRate, } - if self._params.language: - settings["lang"] = self.language_to_service_language(self._params.language) or "eng" - if self._params.segment is not None: - settings["segment"] = self._params.segment + if is_given(self._settings.language): + params["lang"] = self._settings.language + if is_given(self._settings.segment): + params["segment"] = self._settings.segment - if self._model == "arcana": - if self._params.repetition_penalty is not None: - settings["repetition_penalty"] = self._params.repetition_penalty - if self._params.temperature is not None: - settings["temperature"] = self._params.temperature - if self._params.top_p is not None: - settings["top_p"] = self._params.top_p + if self._settings.model == "arcana": + if is_given(self._settings.repetition_penalty): + params["repetition_penalty"] = self._settings.repetition_penalty + if is_given(self._settings.temperature): + params["temperature"] = self._settings.temperature + if is_given(self._settings.top_p): + params["top_p"] = self._settings.top_p else: # mistv2/mist - if self._params.speed_alpha is not None: - settings["speedAlpha"] = self._params.speed_alpha - if self._params.reduce_latency is not None: - settings["reduceLatency"] = self._params.reduce_latency - if self._params.pause_between_brackets is not None: - settings["pauseBetweenBrackets"] = json.dumps(self._params.pause_between_brackets) - if self._params.phonemize_between_brackets is not None: - settings["phonemizeBetweenBrackets"] = json.dumps( - self._params.phonemize_between_brackets + if is_given(self._settings.speedAlpha): + params["speedAlpha"] = self._settings.speedAlpha + if is_given(self._settings.reduceLatency): + params["reduceLatency"] = self._settings.reduceLatency + if is_given(self._settings.pauseBetweenBrackets): + params["pauseBetweenBrackets"] = json.dumps(self._settings.pauseBetweenBrackets) + if is_given(self._settings.phonemizeBetweenBrackets): + params["phonemizeBetweenBrackets"] = json.dumps( + self._settings.phonemizeBetweenBrackets ) - if self._params.no_text_normalization is not None: - settings["noTextNormalization"] = json.dumps(self._params.no_text_normalization) - if self._params.save_oovs is not None: - settings["saveOovs"] = json.dumps(self._params.save_oovs) + if is_given(self._settings.noTextNormalization): + params["noTextNormalization"] = json.dumps(self._settings.noTextNormalization) + if is_given(self._settings.saveOovs): + params["saveOovs"] = json.dumps(self._settings.saveOovs) - return settings - - async def set_model(self, model: str): - """Update the TTS model and reconnect. - - Args: - model: The model name to use for synthesis. - """ - self._model = model - self._settings = self._build_settings() - await super().set_model(model) - if self._websocket: - await self._disconnect() - await self._connect() + return params # A set of Rime-specific helpers for text transformations def SPELL(text: str) -> str: @@ -269,72 +349,20 @@ class RimeTTSService(AudioContextWordTTSService): self._extra_msg_fields["inlineSpeedAlpha"] = ",".join(speed_vals + [str(speed)]) return f"[{text}]" - async def _update_settings(self, settings: Mapping[str, Any]): - """Update service settings and reconnect if necessary. + async def _update_settings(self, update: TTSSettings) -> dict[str, Any]: + """Apply a settings update and reconnect if necessary. Since all settings are WebSocket URL query parameters, any setting change requires reconnecting to apply the new values. """ - prev_settings = self._settings.copy() - await super()._update_settings(settings) + changed = await super()._update_settings(update) - needs_reconnect = False - - if "voice" in settings or "voice_id" in settings: - self._settings["speaker"] = self._voice_id - if prev_settings.get("speaker") != self._voice_id: - logger.info(f"Switching TTS voice to: [{self._voice_id}]") - needs_reconnect = True - - if "model" in settings: - self._settings = self._build_settings() - needs_reconnect = True - - if "language" in settings: - new_lang = self.language_to_service_language(settings["language"]) - if new_lang and new_lang != prev_settings.get("lang"): - logger.info(f"Updating language to: [{new_lang}]") - self._settings["lang"] = new_lang - needs_reconnect = True - - # Arcana params - for key, settings_key in [ - ("repetition_penalty", "repetition_penalty"), - ("temperature", "temperature"), - ("top_p", "top_p"), - ]: - if key in settings and settings[key] != prev_settings.get(settings_key): - self._settings[settings_key] = settings[key] - needs_reconnect = True - - # Mistv2 params - for key, settings_key in [ - ("speed_alpha", "speedAlpha"), - ("reduce_latency", "reduceLatency"), - ]: - if key in settings and settings[key] != prev_settings.get(settings_key): - self._settings[settings_key] = settings[key] - needs_reconnect = True - - # Mistv2 boolean params (need json.dumps) - for key, settings_key in [ - ("pause_between_brackets", "pauseBetweenBrackets"), - ("phonemize_between_brackets", "phonemizeBetweenBrackets"), - ("no_text_normalization", "noTextNormalization"), - ("save_oovs", "saveOovs"), - ]: - if key in settings and json.dumps(settings[key]) != prev_settings.get(settings_key): - self._settings[settings_key] = json.dumps(settings[key]) - needs_reconnect = True - - if "segment" in settings and settings["segment"] != prev_settings.get("segment"): - self._settings["segment"] = settings["segment"] - needs_reconnect = True - - if needs_reconnect and self._websocket: + if changed and self._websocket: await self._disconnect() await self._connect() + return changed + def _build_msg(self, text: str = "") -> dict: """Build JSON message for Rime API.""" msg = {"text": text, "contextId": self.get_active_audio_context_id()} @@ -358,7 +386,7 @@ class RimeTTSService(AudioContextWordTTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings = self._build_settings() + self._settings.samplingRate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): @@ -404,7 +432,8 @@ class RimeTTSService(AudioContextWordTTSService): if self._websocket and self._websocket.state is State.OPEN: return - params = "&".join(f"{k}={v}" for k, v in self._settings.items() if v is not None) + ws_params = self._build_ws_params() + params = "&".join(f"{k}={v}" for k, v in ws_params.items() if v is not None) url = f"{self._url}?{params}" headers = {"Authorization": f"Bearer {self._api_key}"} self._websocket = await websocket_connect(url, additional_headers=headers) @@ -580,6 +609,8 @@ class RimeHttpTTSService(TTSService): Suitable for use cases where streaming is not required. """ + _settings: RimeTTSSettings + class InputParams(BaseModel): """Configuration parameters for Rime HTTP TTS service. @@ -628,20 +659,19 @@ class RimeHttpTTSService(TTSService): self._api_key = api_key self._session = aiohttp_session self._base_url = "https://users.rime.ai/v1/rime-tts" - self._settings = { - "lang": self.language_to_service_language(params.language) + self._settings = RimeTTSSettings( + model=model, + language=self.language_to_service_language(params.language) if params.language else "eng", - "speedAlpha": params.speed_alpha, - "reduceLatency": params.reduce_latency, - "pauseBetweenBrackets": params.pause_between_brackets, - "phonemizeBetweenBrackets": params.phonemize_between_brackets, - } - self.set_voice(voice_id) - self.set_model_name(model) - - if params.inline_speed_alpha: - self._settings["inlineSpeedAlpha"] = params.inline_speed_alpha + speedAlpha=params.speed_alpha, + reduceLatency=params.reduce_latency, + pauseBetweenBrackets=params.pause_between_brackets, + phonemizeBetweenBrackets=params.phonemize_between_brackets, + inlineSpeedAlpha=params.inline_speed_alpha if params.inline_speed_alpha else NOT_GIVEN, + voice=voice_id, + ) + self._sync_model_name_to_metrics() def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -681,10 +711,18 @@ class RimeHttpTTSService(TTSService): "Content-Type": "application/json", } - payload = self._settings.copy() + payload = { + "lang": self._settings.language, + "speedAlpha": self._settings.speedAlpha, + "reduceLatency": self._settings.reduceLatency, + "pauseBetweenBrackets": self._settings.pauseBetweenBrackets, + "phonemizeBetweenBrackets": self._settings.phonemizeBetweenBrackets, + } + if is_given(self._settings.inlineSpeedAlpha): + payload["inlineSpeedAlpha"] = self._settings.inlineSpeedAlpha payload["text"] = text - payload["speaker"] = self._voice_id - payload["modelId"] = self._model_name + payload["speaker"] = self._settings.voice + payload["modelId"] = self._settings.model payload["samplingRate"] = self.sample_rate # Arcana does not support PCM audio @@ -743,6 +781,8 @@ class RimeNonJsonTTSService(InterruptibleTTSService): accepts and returns non-JSON messages. """ + _settings: RimeNonJsonTTSSettings + class InputParams(BaseModel): """Configuration parameters for Rime Non-JSON WebSocket TTS service. @@ -798,28 +838,25 @@ class RimeNonJsonTTSService(InterruptibleTTSService): params = params or RimeNonJsonTTSService.InputParams() self._api_key = api_key self._url = url - self._voice_id = voice_id - self._model = model - self._settings = { - "speaker": voice_id, - "modelId": model, - "audioFormat": audio_format, - "samplingRate": sample_rate, - } - - if params.language: - self._settings["lang"] = self.language_to_service_language(params.language) - if params.segment is not None: - self._settings["segment"] = params.segment - if params.repetition_penalty is not None: - self._settings["repetition_penalty"] = params.repetition_penalty - if params.temperature is not None: - self._settings["temperature"] = params.temperature - if params.top_p is not None: - self._settings["top_p"] = params.top_p + self._settings = RimeNonJsonTTSSettings( + voice=voice_id, + model=model, + audioFormat=audio_format, + samplingRate=sample_rate, + language=self.language_to_service_language(params.language) + if params.language + else NOT_GIVEN, + segment=params.segment if params.segment is not None else NOT_GIVEN, + repetition_penalty=params.repetition_penalty + if params.repetition_penalty is not None + else NOT_GIVEN, + temperature=params.temperature if params.temperature is not None else NOT_GIVEN, + top_p=params.top_p if params.top_p is not None else NOT_GIVEN, + ) + self._sync_model_name_to_metrics() # Add any extra parameters for future compatibility if params.extra: - self._settings.update(params.extra) + self._settings.extra.update(params.extra) self._receive_task = None self._context_id: Optional[str] = None @@ -851,7 +888,7 @@ class RimeNonJsonTTSService(InterruptibleTTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["samplingRate"] = self.sample_rate + self._settings.samplingRate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): @@ -895,8 +932,26 @@ class RimeNonJsonTTSService(InterruptibleTTSService): try: if self._websocket and self._websocket.state is State.OPEN: return - # Build URL with query parameters (only non-None values) - params = "&".join(f"{k}={v}" for k, v in self._settings.items() if v is not None) + # Build URL with query parameters (only given, non-None values) + settings_dict = { + "speaker": self._settings.voice, + "modelId": self._settings.model, + "audioFormat": self._settings.audioFormat, + "samplingRate": self._settings.samplingRate, + } + if is_given(self._settings.language): + settings_dict["lang"] = self._settings.language + if is_given(self._settings.segment): + settings_dict["segment"] = self._settings.segment + if is_given(self._settings.repetition_penalty): + settings_dict["repetition_penalty"] = self._settings.repetition_penalty + if is_given(self._settings.temperature): + settings_dict["temperature"] = self._settings.temperature + if is_given(self._settings.top_p): + settings_dict["top_p"] = self._settings.top_p + # Include extras + settings_dict.update(self._settings.extra) + params = "&".join(f"{k}={v}" for k, v in settings_dict.items() if v is not None) url = f"{self._url}?{params}" headers = {"Authorization": f"Bearer {self._api_key}"} self._websocket = await websocket_connect( @@ -990,68 +1045,17 @@ class RimeNonJsonTTSService(InterruptibleTTSService): except Exception as e: yield ErrorFrame(error=f"Unknown error occurred: {e}") - async def _update_settings(self, settings: Mapping[str, Any]): - """Update service settings and reconnect if necessary. + async def _update_settings(self, update: TTSSettings) -> dict[str, Any]: + """Apply a settings update and reconnect if necessary. Since all settings are WebSocket URL query parameters, any setting change requires reconnecting to apply the new values. """ - needs_reconnect = False + changed = await super()._update_settings(update) - # Track previous values from self._settings only - prev_settings = self._settings.copy() - - # Let parent class handle standard settings (voice, model, language) - await super()._update_settings(settings) - - # Check if voice changed and update settings dict - if "voice" in settings or "voice_id" in settings: - self._settings["speaker"] = self._voice_id - if prev_settings.get("speaker") != self._voice_id: - logger.info(f"Switching TTS voice to: [{self._voice_id}]") - needs_reconnect = True - - # Check if model changed and update settings dict - if "model" in settings: - self._settings["modelId"] = self._model - if prev_settings.get("modelId") != self._model: - logger.info(f"Switching TTS model to: [{self._model}]") - needs_reconnect = True - - # Handle language explicitly - if "language" in settings: - new_lang = self.language_to_service_language(settings["language"]) - if new_lang and new_lang != prev_settings.get("lang"): - logger.info(f"Updating language to: [{new_lang}]") - self._settings["lang"] = new_lang - needs_reconnect = True - - # Check other parameters - for key in ["segment", "repetition_penalty", "temperature", "top_p"]: - if key in settings and settings[key] != prev_settings.get(key): - logger.info(f"Updating {key} to: [{settings[key]}]") - self._settings[key] = settings[key] - needs_reconnect = True - - # Handle extra parameters - for key, value in settings.items(): - if key not in [ - "voice", - "voice_id", - "model", - "language", - "segment", - "repetition_penalty", - "temperature", - "top_p", - ]: - if value != prev_settings.get(key): - logger.info(f"Updating extra parameter {key} to: [{value}]") - self._settings[key] = value - needs_reconnect = True - - # Reconnect if any setting changed - if needs_reconnect: + if changed: logger.debug("Settings changed, reconnecting WebSocket with new parameters") await self._disconnect() await self._connect() + + return changed diff --git a/src/pipecat/services/sambanova/llm.py b/src/pipecat/services/sambanova/llm.py index 047ce0e6c..016e1740d 100644 --- a/src/pipecat/services/sambanova/llm.py +++ b/src/pipecat/services/sambanova/llm.py @@ -84,19 +84,19 @@ class SambaNovaLLMService(OpenAILLMService): # type: ignore Dictionary of parameters for the chat completion request. """ params = { - "model": self.model_name, + "model": self._settings.model, "stream": True, "stream_options": {"include_usage": True}, - "temperature": self._settings["temperature"], - "top_p": self._settings["top_p"], - "max_tokens": self._settings["max_tokens"], - "max_completion_tokens": self._settings["max_completion_tokens"], + "temperature": self._settings.temperature, + "top_p": self._settings.top_p, + "max_tokens": self._settings.max_tokens, + "max_completion_tokens": self._settings.max_completion_tokens, } # Messages, tools, tool_choice params.update(params_from_context) - params.update(self._settings["extra"]) + params.update(self._settings.extra) return params @traced_llm # type: ignore diff --git a/src/pipecat/services/sambanova/stt.py b/src/pipecat/services/sambanova/stt.py index a1cbe8a22..f313f0d7b 100644 --- a/src/pipecat/services/sambanova/stt.py +++ b/src/pipecat/services/sambanova/stt.py @@ -72,7 +72,7 @@ class SambaNovaSTTService(BaseWhisperSTTService): # type: ignore # Build kwargs dict with only set parameters kwargs = { "file": ("audio.wav", audio, "audio/wav"), - "model": self.model_name, + "model": self._settings.model, "response_format": "json", "language": self._language, } diff --git a/src/pipecat/services/sarvam/stt.py b/src/pipecat/services/sarvam/stt.py index e8f2d55ce..0128c1a22 100644 --- a/src/pipecat/services/sarvam/stt.py +++ b/src/pipecat/services/sarvam/stt.py @@ -12,8 +12,8 @@ can handle multiple audio formats for Indian language speech recognition. """ import base64 -from dataclasses import dataclass -from typing import AsyncGenerator, Dict, Literal, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, Dict, Literal, Optional from loguru import logger from pydantic import BaseModel @@ -32,6 +32,7 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.sarvam._sdk import sdk_headers +from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven, is_given from pipecat.services.stt_latency import SARVAM_TTFS_P99 from pipecat.services.stt_service import STTService from pipecat.transcriptions.language import Language, resolve_language @@ -130,6 +131,23 @@ MODEL_CONFIGS: Dict[str, ModelConfig] = { } +@dataclass +class SarvamSTTSettings(STTSettings): + """Settings for the Sarvam STT service. + + Parameters: + prompt: Optional prompt to guide transcription/translation style. + mode: Mode of operation (transcribe, translate, verbatim, etc.). + vad_signals: Enable VAD signals in response. + high_vad_sensitivity: Enable high VAD sensitivity. + """ + + prompt: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + mode: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + vad_signals: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + high_vad_sensitivity: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class SarvamSTTService(STTService): """Sarvam speech-to-text service. @@ -148,6 +166,8 @@ class SarvamSTTService(STTService): ... """ + _settings: SarvamSTTSettings + class InputParams(BaseModel): """Configuration parameters for Sarvam STT service. @@ -228,24 +248,9 @@ class SarvamSTTService(STTService): **kwargs, ) - self.set_model_name(model) self._api_key = api_key - self._language_code: Optional[Language] = params.language - - # Set language string: use provided language or model's default - if params.language: - self._language_string = language_to_sarvam_language(params.language) - else: - self._language_string = self._config.default_language - - self._prompt = params.prompt - - # Set mode: use provided mode or model's default - self._mode = params.mode if params.mode is not None else self._config.default_mode # Store connection parameters - self._vad_signals = params.vad_signals - self._high_vad_sensitivity = params.high_vad_sensitivity self._input_audio_codec = input_audio_codec # Initialize Sarvam SDK client @@ -263,7 +268,20 @@ class SarvamSTTService(STTService): self._socket_client = None self._receive_task = None - if self._vad_signals: + # Resolve mode default from model config + mode = params.mode if params.mode is not None else self._config.default_mode + + self._settings = SarvamSTTSettings( + model=model, + language=params.language, + prompt=params.prompt if params.prompt is not None else NOT_GIVEN, + mode=mode if mode is not None else NOT_GIVEN, + vad_signals=params.vad_signals, + high_vad_sensitivity=params.high_vad_sensitivity, + ) + self._sync_model_name_to_metrics() + + if params.vad_signals: self._register_event_handler("on_speech_started") self._register_event_handler("on_speech_stopped") self._register_event_handler("on_utterance_end") @@ -281,6 +299,12 @@ class SarvamSTTService(STTService): """ return language_to_sarvam_language(language) + def _get_language_string(self) -> Optional[str]: + """Resolve the current language setting to a Sarvam language code string.""" + if self._settings.language: + return language_to_sarvam_language(self._settings.language) + return self._config.default_language + def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -298,50 +322,91 @@ class SarvamSTTService(STTService): await super().process_frame(frame, direction) # Only handle VAD frames when not using Sarvam's VAD signals - if not self._vad_signals: + if not self._settings.vad_signals: if isinstance(frame, VADUserStartedSpeakingFrame): await self._start_metrics() elif isinstance(frame, VADUserStoppedSpeakingFrame): if self._socket_client: await self._socket_client.flush() - async def set_language(self, language: Language): - """Set the recognition language and reconnect. + async def _update_settings(self, update: STTSettings) -> dict[str, Any]: + """Apply a settings update, validate, sync state, and reconnect. Args: - language: The language to use for speech recognition. + update: A :class:`STTSettings` (or ``SarvamSTTSettings``) delta. + + Returns: + Dict mapping changed field names to their previous values. Raises: - ValueError: If called on a model that auto-detects language. + ValueError: If a setting is not supported by the current model. """ - if not self._config.supports_language: - raise ValueError( - f"Model '{self.model_name}' does not support language parameter " - "(auto-detects language)." - ) + # Validate against model capabilities before applying + if is_given(update.language) and update.language is not None: + if not self._config.supports_language: + raise ValueError( + f"Model '{self._settings.model}' does not support language parameter " + "(auto-detects language)." + ) - logger.info(f"Switching STT language to: [{language}]") - self._language_code = language - self._language_string = language_to_sarvam_language(language) - await self._disconnect() - await self._connect() + if isinstance(update, SarvamSTTSettings): + if is_given(update.prompt) and update.prompt is not None: + if not self._config.supports_prompt: + raise ValueError( + f"Model '{self._settings.model}' does not support prompt parameter." + ) + if is_given(update.mode) and update.mode is not None: + if not self._config.supports_mode: + raise ValueError( + f"Model '{self._settings.model}' does not support mode parameter." + ) + + changed = await super()._update_settings(update) + + # TODO: someday we could reconnect here to apply updated settings. + # Code might look something like the below: + # if not changed: + # return changed + + # await self._disconnect() + # await self._connect() + + self._warn_unhandled_updated_settings(changed) + + return changed async def set_prompt(self, prompt: Optional[str]): """Set the transcription/translation prompt and reconnect. + .. deprecated:: + Use ``STTUpdateSettingsFrame(SarvamSTTSettings(prompt=...))`` instead. + Args: prompt: Prompt text to guide transcription/translation style/context. Pass None to clear/disable prompt. Only applicable to models that support prompts. """ + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + f"{self.__class__.__name__}.set_prompt() is deprecated. " + "Use STTUpdateSettingsFrame(SarvamSTTSettings(prompt=...)) instead.", + DeprecationWarning, + stacklevel=2, + ) + if not self._config.supports_prompt: if prompt is not None: - raise ValueError(f"Model '{self.model_name}' does not support prompt parameter.") + raise ValueError( + f"Model '{self._settings.model}' does not support prompt parameter." + ) # If prompt is None and model doesn't support prompts, silently return (no-op) return - logger.info(f"Updating {self.model_name} prompt.") - self._prompt = prompt + logger.info(f"Updating {self._settings.model} prompt.") + self._settings.prompt = prompt await self._disconnect() await self._connect() @@ -422,35 +487,40 @@ class SarvamSTTService(STTService): try: # Build common connection parameters connect_kwargs = { - "model": self.model_name, + "model": self._settings.model, "sample_rate": str(self.sample_rate), } # Enable flush signal when using Pipecat's VAD (not Sarvam's) so that # the flush() call on user-stopped-speaking is honored by the server. - if not self._vad_signals: + if not self._settings.vad_signals: connect_kwargs["flush_signal"] = "true" # Only send vad parameters when explicitly set (avoid overriding server defaults) - if self._vad_signals is not None: - connect_kwargs["vad_signals"] = "true" if self._vad_signals else "false" - if self._high_vad_sensitivity is not None: + if self._settings.vad_signals is not None: + connect_kwargs["vad_signals"] = "true" if self._settings.vad_signals else "false" + if self._settings.high_vad_sensitivity is not None: connect_kwargs["high_vad_sensitivity"] = ( - "true" if self._high_vad_sensitivity else "false" + "true" if self._settings.high_vad_sensitivity else "false" ) # Add language_code for models that support it - if self._language_string is not None: - connect_kwargs["language_code"] = self._language_string + language_string = self._get_language_string() + if language_string is not None: + connect_kwargs["language_code"] = language_string # Add mode for models that support it - if self._config.supports_mode and self._mode is not None: - connect_kwargs["mode"] = self._mode + if self._config.supports_mode and is_given(self._settings.mode): + connect_kwargs["mode"] = self._settings.mode # Prompt support differs across sarvamai versions. Prefer connect-time prompt # when available and gracefully degrade if the SDK doesn't accept it. - if self._prompt is not None and self._config.supports_prompt: - connect_kwargs["prompt"] = self._prompt + if ( + is_given(self._settings.prompt) + and self._settings.prompt is not None + and self._config.supports_prompt + ): + connect_kwargs["prompt"] = self._settings.prompt def _connect_with_sdk_headers(connect_fn, **kwargs): # Different SDK versions may use different kwarg names. @@ -491,10 +561,14 @@ class SarvamSTTService(STTService): self._socket_client = await self._websocket_context.__aenter__() # Fallback for SDKs that support runtime prompt updates. - if self._prompt is not None and self._config.supports_prompt: + if ( + is_given(self._settings.prompt) + and self._settings.prompt is not None + and self._config.supports_prompt + ): prompt_setter = getattr(self._socket_client, "set_prompt", None) if callable(prompt_setter): - await prompt_setter(self._prompt) + await prompt_setter(self._settings.prompt) # Register event handler for incoming messages def _message_handler(message): @@ -592,10 +666,12 @@ class SarvamSTTService(STTService): # Prefer language from message (auto-detected for translate models). Fallback to configured. if language_code: language = self._map_language_code_to_enum(language_code) - elif self._language_string: - language = self._map_language_code_to_enum(self._language_string) else: - language = Language.HI_IN + language_string = self._get_language_string() + if language_string: + language = self._map_language_code_to_enum(language_string) + else: + language = Language.HI_IN # Emit utterance end event await self._call_event_handler("on_utterance_end") diff --git a/src/pipecat/services/sarvam/tts.py b/src/pipecat/services/sarvam/tts.py index 753293c75..191689f5a 100644 --- a/src/pipecat/services/sarvam/tts.py +++ b/src/pipecat/services/sarvam/tts.py @@ -40,9 +40,9 @@ See https://docs.sarvam.ai/api-reference-docs/text-to-speech/stream for full API import asyncio import base64 import json -from dataclasses import dataclass +from dataclasses import dataclass, field from enum import Enum -from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Tuple +from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple import aiohttp from loguru import logger @@ -62,6 +62,7 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.sarvam._sdk import sdk_headers +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven, is_given from pipecat.services.tts_service import InterruptibleTTSService, TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -244,6 +245,78 @@ def language_to_sarvam_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +@dataclass +class SarvamHttpTTSSettings(TTSSettings): + """Settings for Sarvam HTTP TTS service. + + Parameters: + language: Sarvam language code. + enable_preprocessing: Whether to enable text preprocessing. Defaults to False. + **Note:** Always enabled for bulbul:v3-beta (cannot be disabled). + pace: Speech pace multiplier. Defaults to 1.0. + - bulbul:v2: Range 0.3 to 3.0 + - bulbul:v3-beta: Range 0.5 to 2.0 + pitch: Voice pitch adjustment (-0.75 to 0.75). Defaults to 0.0. + **Note:** Only supported for bulbul:v2. Ignored for v3 models. + loudness: Volume multiplier (0.3 to 3.0). Defaults to 1.0. + **Note:** Only supported for bulbul:v2. Ignored for v3 models. + temperature: Controls output randomness for bulbul:v3-beta (0.01 to 1.0). + Lower values = more deterministic, higher = more random. Defaults to 0.6. + **Note:** Only supported for bulbul:v3-beta. Ignored for v2. + sample_rate: Audio sample rate. + """ + + language: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + enable_preprocessing: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + pace: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + pitch: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + loudness: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + temperature: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + sarvam_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + +@dataclass +class SarvamTTSSettings(TTSSettings): + """Settings for Sarvam WebSocket TTS service. + + Parameters: + target_language_code: Sarvam language code. + speech_sample_rate: Audio sample rate as string. + enable_preprocessing: Enable text preprocessing. Defaults to False. + **Note:** Always enabled for bulbul:v3-beta. + min_buffer_size: Minimum characters to buffer before generating audio. + Lower values reduce latency but may affect quality. Defaults to 50. + max_chunk_length: Maximum characters processed in a single chunk. + Controls memory usage and processing efficiency. Defaults to 150. + output_audio_codec: Audio codec format. Options: linear16, mulaw, alaw, + opus, flac, aac, wav, mp3. Defaults to "linear16". + output_audio_bitrate: Audio bitrate (32k, 64k, 96k, 128k, 192k). + Defaults to "128k". + pace: Speech pace multiplier. Defaults to 1.0. + - bulbul:v2: Range 0.3 to 3.0 + - bulbul:v3-beta: Range 0.5 to 2.0 + pitch: Voice pitch adjustment (-0.75 to 0.75). Defaults to 0.0. + **Note:** Only supported for bulbul:v2. Ignored for v3 models. + loudness: Volume multiplier (0.3 to 3.0). Defaults to 1.0. + **Note:** Only supported for bulbul:v2. Ignored for v3 models. + temperature: Controls output randomness for bulbul:v3-beta (0.01 to 1.0). + Lower = more deterministic, higher = more random. Defaults to 0.6. + **Note:** Only supported for bulbul:v3-beta. Ignored for v2. + """ + + target_language_code: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + speech_sample_rate: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + enable_preprocessing: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + min_buffer_size: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + max_chunk_length: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + output_audio_codec: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + output_audio_bitrate: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + pace: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + pitch: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + loudness: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + temperature: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class SarvamHttpTTSService(TTSService): """Text-to-Speech service using Sarvam AI's API. @@ -296,6 +369,8 @@ class SarvamHttpTTSService(TTSService): ) """ + _settings: SarvamHttpTTSSettings + class InputParams(BaseModel): """Input parameters for Sarvam TTS configuration. @@ -383,14 +458,14 @@ class SarvamHttpTTSService(TTSService): if sample_rate is None: sample_rate = self._config.default_sample_rate - super().__init__(sample_rate=sample_rate, **kwargs) - params = params or SarvamHttpTTSService.InputParams() # Set default voice based on model if not specified if voice_id is None: voice_id = self._config.default_speaker + super().__init__(sample_rate=sample_rate, **kwargs) + self._api_key = api_key self._base_url = base_url self._session = aiohttp_session @@ -403,36 +478,35 @@ class SarvamHttpTTSService(TTSService): pace = max(pace_min, min(pace_max, pace)) # Build base settings - self._settings = { - "language": ( + self._settings = SarvamHttpTTSSettings( + language=( self.language_to_service_language(params.language) if params.language else "en-IN" ), - "enable_preprocessing": ( + enable_preprocessing=( True if self._config.preprocessing_always_enabled else params.enable_preprocessing ), - "pace": pace, - "model": model, - } + pace=pace, + model=model, + voice=voice_id, + ) + self._sync_model_name_to_metrics() # Add parameters based on model support if self._config.supports_pitch: - self._settings["pitch"] = params.pitch + self._settings.pitch = params.pitch elif params.pitch != 0.0: logger.warning(f"pitch parameter is ignored for {model}") if self._config.supports_loudness: - self._settings["loudness"] = params.loudness + self._settings.loudness = params.loudness elif params.loudness != 1.0: logger.warning(f"loudness parameter is ignored for {model}") if self._config.supports_temperature: - self._settings["temperature"] = params.temperature + self._settings.temperature = params.temperature elif params.temperature != 0.6: logger.warning(f"temperature parameter is ignored for {model}") - self.set_model_name(model) - self.set_voice(voice_id) - def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -459,7 +533,7 @@ class SarvamHttpTTSService(TTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["sample_rate"] = self.sample_rate + self._settings.sarvam_sample_rate = self.sample_rate @traced_tts async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]: @@ -480,21 +554,25 @@ class SarvamHttpTTSService(TTSService): # Build payload with common parameters payload = { "text": text, - "target_language_code": self._settings["language"], - "speaker": self._voice_id, + "target_language_code": self._settings.language, + "speaker": self._settings.voice, "sample_rate": self.sample_rate, - "enable_preprocessing": self._settings["enable_preprocessing"], - "model": self._model_name, - "pace": self._settings.get("pace", 1.0), + "enable_preprocessing": self._settings.enable_preprocessing, + "model": self._settings.model, + "pace": self._settings.pace if is_given(self._settings.pace) else 1.0, } # Add model-specific parameters based on config if self._config.supports_pitch: - payload["pitch"] = self._settings.get("pitch", 0.0) + payload["pitch"] = self._settings.pitch if is_given(self._settings.pitch) else 0.0 if self._config.supports_loudness: - payload["loudness"] = self._settings.get("loudness", 1.0) + payload["loudness"] = ( + self._settings.loudness if is_given(self._settings.loudness) else 1.0 + ) if self._config.supports_temperature: - payload["temperature"] = self._settings.get("temperature", 0.6) + payload["temperature"] = ( + self._settings.temperature if is_given(self._settings.temperature) else 0.6 + ) headers = { "api-subscription-key": self._api_key, @@ -605,6 +683,8 @@ class SarvamTTSService(InterruptibleTTSService): See https://docs.sarvam.ai/api-reference-docs/text-to-speech/stream for API details. """ + _settings: SarvamTTSSettings + class InputParams(BaseModel): """Configuration parameters for Sarvam TTS WebSocket service. @@ -729,6 +809,10 @@ class SarvamTTSService(InterruptibleTTSService): if sample_rate is None: sample_rate = self._config.default_sample_rate + # Set default voice based on model if not specified + if voice_id is None: + voice_id = self._config.default_speaker + # Initialize parent class first super().__init__( aggregate_sentences=aggregate_sentences, @@ -740,15 +824,9 @@ class SarvamTTSService(InterruptibleTTSService): ) params = params or SarvamTTSService.InputParams() - # Set default voice based on model if not specified - if voice_id is None: - voice_id = self._config.default_speaker - # WebSocket endpoint URL with model query parameter self._websocket_url = f"{url}?model={model}" self._api_key = api_key - self.set_model_name(model) - self.set_voice(voice_id) # Validate and clamp pace to model's valid range pace = params.pace @@ -758,36 +836,37 @@ class SarvamTTSService(InterruptibleTTSService): pace = max(pace_min, min(pace_max, pace)) # Build base settings - self._settings = { - "target_language_code": ( + self._settings = SarvamTTSSettings( + target_language_code=( self.language_to_service_language(params.language) if params.language else "en-IN" ), - "speaker": voice_id, - "speech_sample_rate": str(sample_rate), - "enable_preprocessing": ( + speech_sample_rate=str(sample_rate), + enable_preprocessing=( True if self._config.preprocessing_always_enabled else params.enable_preprocessing ), - "min_buffer_size": params.min_buffer_size, - "max_chunk_length": params.max_chunk_length, - "output_audio_codec": params.output_audio_codec, - "output_audio_bitrate": params.output_audio_bitrate, - "pace": pace, - "model": model, - } + min_buffer_size=params.min_buffer_size, + max_chunk_length=params.max_chunk_length, + output_audio_codec=params.output_audio_codec, + output_audio_bitrate=params.output_audio_bitrate, + pace=pace, + model=model, + voice=voice_id, + ) + self._sync_model_name_to_metrics() # Add parameters based on model support if self._config.supports_pitch: - self._settings["pitch"] = params.pitch + self._settings.pitch = params.pitch elif params.pitch != 0.0: logger.warning(f"pitch parameter is ignored for {model}") if self._config.supports_loudness: - self._settings["loudness"] = params.loudness + self._settings.loudness = params.loudness elif params.loudness != 1.0: logger.warning(f"loudness parameter is ignored for {model}") if self._config.supports_temperature: - self._settings["temperature"] = params.temperature + self._settings.temperature = params.temperature elif params.temperature != 0.6: logger.warning(f"temperature parameter is ignored for {model}") @@ -823,7 +902,7 @@ class SarvamTTSService(InterruptibleTTSService): await super().start(frame) # WebSocket API expects sample rate as string - self._settings["speech_sample_rate"] = str(self.sample_rate) + self._settings.speech_sample_rate = str(self.sample_rate) await self._connect() async def stop(self, frame: EndFrame): @@ -870,14 +949,15 @@ class SarvamTTSService(InterruptibleTTSService): if isinstance(frame, (LLMFullResponseEndFrame, EndFrame)): await self.flush_audio() - async def _update_settings(self, settings: Mapping[str, Any]): - """Update service settings and reconnect if voice changed.""" - prev_voice = self._voice_id - await super()._update_settings(settings) - if not prev_voice == self._voice_id: - logger.info(f"Switching TTS voice to: [{self._voice_id}]") + async def _update_settings(self, update: TTSSettings) -> dict[str, Any]: + """Apply a settings update and resend config if voice changed.""" + changed = await super()._update_settings(update) + + if changed: await self._send_config() + return changed + async def _connect(self): """Connect to Sarvam WebSocket and start background tasks.""" await super()._connect() @@ -934,9 +1014,27 @@ class SarvamTTSService(InterruptibleTTSService): """Send initial configuration message.""" if not self._websocket: raise Exception("WebSocket not connected") - self._settings["speaker"] = self._voice_id - logger.debug(f"Config being sent is {self._settings}") - config_message = {"type": "config", "data": self._settings} + # Build config dict for the API + config_data = { + "target_language_code": self._settings.target_language_code, + "speaker": self._settings.voice, + "speech_sample_rate": self._settings.speech_sample_rate, + "enable_preprocessing": self._settings.enable_preprocessing, + "min_buffer_size": self._settings.min_buffer_size, + "max_chunk_length": self._settings.max_chunk_length, + "output_audio_codec": self._settings.output_audio_codec, + "output_audio_bitrate": self._settings.output_audio_bitrate, + "pace": self._settings.pace, + "model": self._settings.model, + } + if is_given(self._settings.pitch): + config_data["pitch"] = self._settings.pitch + if is_given(self._settings.loudness): + config_data["loudness"] = self._settings.loudness + if is_given(self._settings.temperature): + config_data["temperature"] = self._settings.temperature + logger.debug(f"Config being sent is {config_data}") + config_message = {"type": "config", "data": config_data} try: await self._websocket.send(json.dumps(config_message)) diff --git a/src/pipecat/services/settings.py b/src/pipecat/services/settings.py new file mode 100644 index 000000000..4664ecd39 --- /dev/null +++ b/src/pipecat/services/settings.py @@ -0,0 +1,333 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Settings infrastructure for Pipecat AI services. + +This module provides dataclass-based settings objects for service configuration. +Each service type has a corresponding settings class (e.g. ``TTSSettings``, +``LLMSettings``) whose fields use the ``NOT_GIVEN`` sentinel to distinguish +"leave unchanged" from an explicit ``None``. + +Key concepts: + +- **NOT_GIVEN sentinel**: A value meaning "this field was not provided in the + update". Distinct from ``None`` (which may be a valid value for a setting). +- **Settings as both state and delta**: The same class is used for the + service's current settings *and* for update objects. Fields set to + ``NOT_GIVEN`` are simply skipped when applying an update. +- **apply_update**: Applies a delta onto a target settings object and returns + a dict mapping each changed field name to its previous value. +- **from_mapping**: Constructs a settings object from a plain dict, + supporting field aliases (e.g. ``"voice_id"`` → ``"voice"``). +- **Extras**: Unknown keys land in the ``extra`` dict so services that have + non-standard settings don't lose data. +""" + +from __future__ import annotations + +import copy +from dataclasses import dataclass, field, fields +from typing import TYPE_CHECKING, Any, ClassVar, Dict, Mapping, Optional, Type, TypeVar + +from loguru import logger + +from pipecat.transcriptions.language import Language + +if TYPE_CHECKING: + from pipecat.turns.user_turn_completion_mixin import UserTurnCompletionConfig + +# --------------------------------------------------------------------------- +# NOT_GIVEN sentinel +# --------------------------------------------------------------------------- + + +class _NotGiven: + """Sentinel indicating a settings field was not provided. + + ``NOT_GIVEN`` means "the caller did not supply this value" — distinct from + ``None``, which may be a legitimate setting value. It is used as the + default for every settings field so that ``apply_update`` can tell which + fields the caller actually wants to change. + """ + + _instance: Optional[_NotGiven] = None + + def __new__(cls) -> _NotGiven: + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __repr__(self) -> str: + return "NOT_GIVEN" + + def __bool__(self) -> bool: + return False + + +NOT_GIVEN: _NotGiven = _NotGiven() +"""Singleton sentinel meaning "this field was not included in the update".""" + + +def is_given(value: Any) -> bool: + """Check whether a value was explicitly provided (i.e. is not ``NOT_GIVEN``). + + Args: + value: The value to check. + + Returns: + ``True`` if *value* is anything other than ``NOT_GIVEN``. + """ + return not isinstance(value, _NotGiven) + + +# --------------------------------------------------------------------------- +# Base ServiceSettings +# --------------------------------------------------------------------------- + +_S = TypeVar("_S", bound="ServiceSettings") + + +@dataclass +class ServiceSettings: + """Base class for runtime-updatable service settings. + + These settings represent the subset of a service's configuration that can + be changed **while the pipeline is running** (e.g. switching the model or + changing the voice). They are *not* meant to capture every constructor + parameter — only those that support live updates via + ``*UpdateSettingsFrame``. + + Every AI service type (LLM, TTS, STT) extends this with its own fields. + Fields default to ``NOT_GIVEN`` so that an instance can represent either + the full current state **or** a sparse update delta. Note that in the full + current state, **all fields will be given** (i.e. ``NOT_GIVEN`` is reserved + for update deltas). + + Parameters: + model: The model identifier used by the service. + extra: Overflow dict for service-specific keys that don't map to a + declared field. + """ + + # -- common fields ------------------------------------------------------- + + model: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + """AI model identifier (e.g. ``"gpt-4o"``, ``"eleven_turbo_v2_5"``).""" + + extra: Dict[str, Any] = field(default_factory=dict) + """Catch-all for service-specific keys that have no declared field.""" + + # -- class-level configuration ------------------------------------------- + + _aliases: ClassVar[Dict[str, str]] = {} + """Map of alternative key names to canonical field names. + + For example ``{"voice_id": "voice"}`` lets callers use either spelling. + Subclasses should override this as needed. + """ + + # -- public API ---------------------------------------------------------- + + def given_fields(self) -> Dict[str, Any]: + """Return a dict of only the fields that were explicitly provided. + + Skips ``NOT_GIVEN`` values and the ``extra`` field itself. Entries + from ``extra`` are included at the top level. + + Returns: + Dictionary mapping field names to their provided values. + """ + result: Dict[str, Any] = {} + for f in fields(self): + if f.name == "extra": + continue + val = getattr(self, f.name) + if is_given(val): + result[f.name] = val + result.update(self.extra) + return result + + def apply_update(self: _S, update: _S) -> Dict[str, Any]: + """Apply *update* onto this settings object, returning changed fields. + + Only fields in *update* that are **given** (i.e. not ``NOT_GIVEN``) + are considered. A field is "changed" if its new value differs from + the current value. + + The ``extra`` dicts are merged: keys present in the update overwrite + keys in the target. + + Args: + update: A settings object of the same type containing the delta. + + Returns: + A dict mapping each changed field name to its **pre-update** value. + Use ``changed.keys()`` for the set of names, or index with + ``changed["field"]`` to inspect the old value. + + Examples:: + + current = TTSSettings(voice="alice", language="en") + delta = TTSSettings(voice="bob") + changed = current.apply_update(delta) + # changed == {"voice": "alice"} + # current.voice == "bob", current.language == "en" + """ + changed: Dict[str, Any] = {} + for f in fields(self): + if f.name == "extra": + continue + new_val = getattr(update, f.name) + if not is_given(new_val): + continue + old_val = getattr(self, f.name) + if old_val != new_val: + setattr(self, f.name, new_val) + changed[f.name] = old_val + + # Merge extra + for key, new_val in update.extra.items(): + old_val = self.extra.get(key, NOT_GIVEN) + if old_val != new_val: + self.extra[key] = new_val + changed[key] = old_val + + return changed + + @classmethod + def from_mapping(cls: Type[_S], settings: Mapping[str, Any]) -> _S: + """Construct a settings object from a plain dictionary. + + This exists for backward compatibility with code that passes plain + dicts via ``*UpdateSettingsFrame(settings={...})``. + + Keys are matched to dataclass fields by name. Keys listed in + ``_aliases`` are translated to their canonical name first. Any + remaining unrecognized keys are placed into ``extra``. + + Args: + settings: A dictionary of setting names to values. + + Returns: + A new settings instance with the corresponding fields populated. + + Examples:: + + update = TTSSettings.from_mapping({"voice_id": "alice", "speed": 1.2}) + # update.voice == "alice" (via alias) + # update.extra == {"speed": 1.2} + """ + field_names = {f.name for f in fields(cls)} - {"extra"} + kwargs: Dict[str, Any] = {} + extra: Dict[str, Any] = {} + + for key, value in settings.items(): + # Resolve aliases first + canonical = cls._aliases.get(key, key) + if canonical in field_names: + kwargs[canonical] = value + else: + extra[key] = value + + instance = cls(**kwargs) + instance.extra = extra + return instance + + def copy(self: _S) -> _S: + """Return a deep copy of this settings instance. + + Returns: + A new settings object with the same field values. + """ + return copy.deepcopy(self) + + +# --------------------------------------------------------------------------- +# Service-specific settings +# --------------------------------------------------------------------------- + + +@dataclass +class LLMSettings(ServiceSettings): + """Runtime-updatable settings for LLM services. + + See ``ServiceSettings`` for the general concept. + + Parameters: + model: LLM model identifier. + temperature: Sampling temperature. + max_tokens: Maximum tokens to generate. + top_p: Nucleus sampling probability. + top_k: Top-k sampling parameter. + frequency_penalty: Frequency penalty. + presence_penalty: Presence penalty. + seed: Random seed for reproducibility. + filter_incomplete_user_turns: Enable LLM-based turn completion detection + to suppress bot responses when the user was cut off mid-thought. + See ``examples/foundational/22-filter-incomplete-turns.py`` and + ``UserTurnCompletionLLMServiceMixin``. + user_turn_completion_config: Configuration for turn completion behavior + when ``filter_incomplete_user_turns`` is enabled. Controls timeouts + and prompts for incomplete turns. + """ + + temperature: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + max_tokens: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + top_p: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + top_k: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + frequency_penalty: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + presence_penalty: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + seed: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + filter_incomplete_user_turns: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + user_turn_completion_config: UserTurnCompletionConfig | _NotGiven = field( + default_factory=lambda: NOT_GIVEN + ) + + +@dataclass +class TTSSettings(ServiceSettings): + """Runtime-updatable settings for TTS services. + + See ``ServiceSettings`` for the general concept. + + Parameters: + model: TTS model identifier. + voice: Voice identifier or name. + language: Language for speech synthesis. The union type reflects the + *input* side: callers may pass a ``Language`` enum or a raw string. + However, the **stored** value is always a service-specific string + — ``TTSService._update_settings`` converts ``Language`` enums via + ``language_to_service_language()`` before writing, and ``__init__`` + methods do the same at construction time. Code that reads + ``self._settings.language`` after initialisation can treat it as + ``str``. + """ + + voice: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + language: Language | str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + _aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice"} + + +@dataclass +class STTSettings(ServiceSettings): + """Runtime-updatable settings for STT services. + + See ``ServiceSettings`` for the general concept. + + Parameters: + model: STT model identifier. + language: Language for speech recognition. The union type reflects the + *input* side: callers may pass a ``Language`` enum or a raw string. + However, the **stored** value is always a service-specific string + — ``STTService._update_settings`` converts ``Language`` enums via + ``language_to_service_language()`` before writing, and ``__init__`` + methods do the same at construction time. Code that reads + ``self._settings.language`` after initialisation can treat it as + ``str``. + """ + + language: Language | str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) diff --git a/src/pipecat/services/soniox/stt.py b/src/pipecat/services/soniox/stt.py index c9184ba4c..1e4b49705 100644 --- a/src/pipecat/services/soniox/stt.py +++ b/src/pipecat/services/soniox/stt.py @@ -8,7 +8,8 @@ import json import time -from typing import AsyncGenerator, List, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, List, Optional from loguru import logger from pydantic import BaseModel @@ -23,6 +24,7 @@ from pipecat.frames.frames import ( VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven, is_given from pipecat.services.stt_latency import SONIOX_TTFS_P99 from pipecat.services.stt_service import WebsocketSTTService from pipecat.transcriptions.language import Language @@ -134,6 +136,17 @@ def _prepare_language_hints( return list(set(prepared_languages)) +@dataclass +class SonioxSTTSettings(STTSettings): + """Settings for Soniox STT service. + + Parameters: + input_params: Soniox ``SonioxInputParams`` for detailed configuration. + """ + + input_params: SonioxInputParams | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class SonioxSTTService(WebsocketSTTService): """Speech-to-Text service using Soniox's WebSocket API. @@ -144,6 +157,8 @@ class SonioxSTTService(WebsocketSTTService): For complete API documentation, see: https://soniox.com/docs/speech-to-text/api-reference/websocket-api """ + _settings: SonioxSTTSettings + def __init__( self, *, @@ -180,10 +195,14 @@ class SonioxSTTService(WebsocketSTTService): self._api_key = api_key self._url = url - self.set_model_name(params.model) - self._params = params self._vad_force_turn_endpoint = vad_force_turn_endpoint + self._settings = SonioxSTTSettings( + model=params.model, + input_params=params, + ) + self._sync_model_name_to_metrics() + self._final_transcription_buffer = [] self._last_tokens_received: Optional[float] = None @@ -198,6 +217,47 @@ class SonioxSTTService(WebsocketSTTService): await super().start(frame) await self._connect() + async def _update_settings(self, update: SonioxSTTSettings) -> dict[str, Any]: + """Apply a settings update, keeping ``input_params`` in sync. + + Top-level ``model`` is the source of truth. When it is given in + *update* its value is propagated into ``input_params``. When only + ``input_params`` is given, its ``model`` is propagated *up* to the + top-level field. + + Settings are stored but not applied to the active connection. + + Args: + update: A settings delta. + + Returns: + Dict mapping changed field names to their previous values. + """ + model_given = is_given(getattr(update, "model", NOT_GIVEN)) + + changed = await super()._update_settings(update) + + if not changed: + return changed + + # --- Sync model -------------------------------------------------- + if model_given: + # Top-level model wins → push into input_params. + self._settings.input_params.model = self._settings.model + elif "input_params" in changed and self._settings.input_params.model is not None: + # Only input_params was given → pull model up. + self._settings.model = self._settings.input_params.model + self._sync_model_name_to_metrics() + + # TODO: someday we could reconnect here to apply updated settings. + # Code might look something like the below: + # await self._disconnect() + # await self._connect() + + self._warn_unhandled_updated_settings(changed) + + return changed + async def stop(self, frame: EndFrame): """Stop the Soniox STT websocket connection. @@ -311,24 +371,26 @@ class SonioxSTTService(WebsocketSTTService): # Either one or the other is required. enable_endpoint_detection = not self._vad_force_turn_endpoint - context = self._params.context + params = self._settings.input_params + + context = params.context if isinstance(context, SonioxContextObject): context = context.model_dump() # Send the initial configuration message. config = { "api_key": self._api_key, - "model": self._model_name, - "audio_format": self._params.audio_format, - "num_channels": self._params.num_channels or 1, + "model": self._settings.model, + "audio_format": params.audio_format, + "num_channels": params.num_channels or 1, "enable_endpoint_detection": enable_endpoint_detection, "sample_rate": self.sample_rate, - "language_hints": _prepare_language_hints(self._params.language_hints), - "language_hints_strict": self._params.language_hints_strict, + "language_hints": _prepare_language_hints(params.language_hints), + "language_hints_strict": params.language_hints_strict, "context": context, - "enable_speaker_diarization": self._params.enable_speaker_diarization, - "enable_language_identification": self._params.enable_language_identification, - "client_reference_id": self._params.client_reference_id, + "enable_speaker_diarization": params.enable_speaker_diarization, + "enable_language_identification": params.enable_language_identification, + "client_reference_id": params.client_reference_id, } # Send the configuration message. diff --git a/src/pipecat/services/speechmatics/stt.py b/src/pipecat/services/speechmatics/stt.py index 72f3f3990..4a476c01f 100644 --- a/src/pipecat/services/speechmatics/stt.py +++ b/src/pipecat/services/speechmatics/stt.py @@ -8,8 +8,10 @@ import asyncio import os +import warnings +from dataclasses import dataclass, field from enum import Enum -from typing import Any, AsyncGenerator +from typing import Any, AsyncGenerator, ClassVar from dotenv import load_dotenv from loguru import logger @@ -31,6 +33,7 @@ from pipecat.frames.frames import ( VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven, is_given from pipecat.services.stt_latency import SPEECHMATICS_TTFS_P99 from pipecat.services.stt_service import STTService from pipecat.transcriptions.language import Language, resolve_language @@ -80,6 +83,83 @@ class TurnDetectionMode(str, Enum): SMART_TURN = "smart_turn" +@dataclass +class SpeechmaticsSTTSettings(STTSettings): + """Settings for Speechmatics STT service. + + See ``SpeechmaticsSTTService.InputParams`` for detailed descriptions of each field. + + Parameters: + model: The operating point / model name. + domain: Domain for Speechmatics API. + turn_detection_mode: Endpoint handling mode. + speaker_active_format: Formatter for active speaker ID. + speaker_passive_format: Formatter for passive speaker ID. + focus_speakers: List of speaker IDs to focus on. + ignore_speakers: List of speaker IDs to ignore. + focus_mode: Speaker focus mode for diarization. + known_speakers: List of known speaker labels and identifiers. + additional_vocab: List of additional vocabulary entries. + audio_encoding: Audio encoding format. + operating_point: Operating point for accuracy vs. latency. + max_delay: Maximum delay in seconds for transcription. + end_of_utterance_silence_trigger: Maximum delay for end of utterance trigger. + end_of_utterance_max_delay: Maximum delay for end of utterance. + punctuation_overrides: Punctuation overrides. + include_partials: Include partial segment fragments. + split_sentences: Emit finalized sentences mid-turn. + enable_diarization: Enable speaker diarization. + speaker_sensitivity: Diarization sensitivity. + max_speakers: Maximum number of speakers to detect. + prefer_current_speaker: Prefer current speaker ID. + extra_params: Extra parameters for the STT engine. + """ + + domain: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + turn_detection_mode: TurnDetectionMode | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + speaker_active_format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + speaker_passive_format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + focus_speakers: list[str] | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + ignore_speakers: list[str] | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + focus_mode: SpeakerFocusMode | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + known_speakers: list[SpeakerIdentifier] | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + additional_vocab: list[AdditionalVocabEntry] | _NotGiven = field( + default_factory=lambda: NOT_GIVEN + ) + audio_encoding: AudioEncoding | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + operating_point: OperatingPoint | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + max_delay: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + end_of_utterance_silence_trigger: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + end_of_utterance_max_delay: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + punctuation_overrides: dict[str, Any] | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + include_partials: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + split_sentences: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + enable_diarization: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + speaker_sensitivity: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + max_speakers: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + prefer_current_speaker: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + extra_params: dict[str, Any] | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + #: Fields that can be updated on a live connection via the Speechmatics + #: diarization-config API — no reconnect needed. + HOT_FIELDS: ClassVar[frozenset[str]] = frozenset( + { + "focus_speakers", + "ignore_speakers", + "focus_mode", + } + ) + + #: Fields that are purely local (formatting templates) — no reconnect + #: and no API call needed. + LOCAL_FIELDS: ClassVar[frozenset[str]] = frozenset( + { + "speaker_active_format", + "speaker_passive_format", + } + ) + + class SpeechmaticsSTTService(STTService): """Speechmatics STT service implementation. @@ -98,6 +178,8 @@ class SpeechmaticsSTTService(STTService): ... """ + _settings: SpeechmaticsSTTSettings + # Export related classes as class attributes TurnDetectionMode = TurnDetectionMode AudioEncoding = AudioEncoding @@ -337,31 +419,57 @@ class SpeechmaticsSTTService(STTService): # Deprecation check self._check_deprecated_args(kwargs, params) - # Voice agent + # Output formatting defaults + speaker_active_format = params.speaker_active_format + if speaker_active_format is None: + speaker_active_format = ( + "@{speaker_id}: {text}" if params.enable_diarization else "{text}" + ) + speaker_passive_format = params.speaker_passive_format or speaker_active_format + + # Settings — seeded from InputParams + self._settings = SpeechmaticsSTTSettings( + language=params.language, + domain=params.domain, + turn_detection_mode=params.turn_detection_mode, + speaker_active_format=speaker_active_format, + speaker_passive_format=speaker_passive_format, + focus_speakers=params.focus_speakers, + ignore_speakers=params.ignore_speakers, + focus_mode=params.focus_mode, + known_speakers=params.known_speakers, + additional_vocab=params.additional_vocab, + audio_encoding=params.audio_encoding, + operating_point=params.operating_point, + max_delay=params.max_delay, + end_of_utterance_silence_trigger=params.end_of_utterance_silence_trigger, + end_of_utterance_max_delay=params.end_of_utterance_max_delay, + punctuation_overrides=params.punctuation_overrides, + include_partials=params.include_partials, + split_sentences=params.split_sentences, + enable_diarization=params.enable_diarization, + speaker_sensitivity=params.speaker_sensitivity, + max_speakers=params.max_speakers, + prefer_current_speaker=params.prefer_current_speaker, + extra_params=params.extra_params, + ) + + # Build SDK config from settings self._client: VoiceAgentClient | None = None - self._config: VoiceAgentConfig = self._prepare_config(params) + self._config: VoiceAgentConfig = self._build_config() # Outbound frame queue self._outbound_frames: asyncio.Queue[Frame] = asyncio.Queue() - # Output formatting - if params.speaker_active_format is None: - params.speaker_active_format = ( - "@{speaker_id}: {text}" if params.enable_diarization else "{text}" - ) - # Framework options self._enable_vad: bool = self._config.end_of_utterance_mode not in [ EndOfUtteranceMode.FIXED, EndOfUtteranceMode.EXTERNAL, ] - self._speaker_active_format: str = params.speaker_active_format - self._speaker_passive_format: str = ( - params.speaker_passive_format or params.speaker_active_format - ) - # Model + metrics - self.set_model_name(self._config.operating_point.value) + # Model + metrics (operating_point comes from the SDK config/preset) + self._settings.model = self._config.operating_point.value + self._sync_model_name_to_metrics() # Message queue self._stt_msg_queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue() @@ -384,6 +492,64 @@ class SpeechmaticsSTTService(STTService): await super().start(frame) await self._connect() + async def _update_settings(self, update: SpeechmaticsSTTSettings) -> dict[str, Any]: + """Apply settings update, reconnecting only when necessary. + + Fields are classified into three categories (see + ``SpeechmaticsSTTSettings``): + + * **HOT_FIELDS** – diarization speaker settings that can be pushed + to a live Speechmatics connection without reconnecting. + * **LOCAL_FIELDS** – formatting templates evaluated locally; no + reconnect or API call needed. + * Everything else – baked into ``VoiceAgentConfig`` at connection + time and therefore require a full disconnect / reconnect. + + Args: + update: A settings delta. + + Returns: + Dict mapping changed field names to their previous values. + """ + changed = await super()._update_settings(update) + + if not changed: + return changed + + no_reconnect = SpeechmaticsSTTSettings.HOT_FIELDS | SpeechmaticsSTTSettings.LOCAL_FIELDS + needs_reconnect = bool(changed.keys() - no_reconnect) + + if needs_reconnect: + logger.debug(f"{self} settings update requires reconnect: {changed.keys()}") + # Connection-level fields changed — rebuild the SDK config + # from the now-updated self._settings, then reconnect. + self._config = self._build_config() + await self._disconnect() + await self._connect() + elif changed.keys() & SpeechmaticsSTTSettings.HOT_FIELDS: + logger.debug(f"{self} applying hot settings update: {changed.keys()}") + if self._config.enable_diarization: + # Only hot-updatable fields changed — push to the live session. + self._config.speaker_config.focus_speakers = self._settings.focus_speakers + self._config.speaker_config.ignore_speakers = self._settings.ignore_speakers + self._config.speaker_config.focus_mode = self._settings.focus_mode + if self._client: + self._client.update_diarization_config(self._config.speaker_config) + else: + logger.debug( + f"{self} hot settings updated but diarization not enabled: {changed.keys()}. ignoring." + ) + # Diarization not enabled — the new settings will take effect + # if/when diarization is enabled, which does require a reconnect. + elif changed.keys() & SpeechmaticsSTTSettings.LOCAL_FIELDS: + logger.debug( + f"{self} local settings update, no special action required: {changed.keys()}" + ) + # Only local fields changed — no need to push to the STT engine, + # the new settings will take effect immediately. + + return changed + async def stop(self, frame: EndFrame): """Called when the session ends.""" await super().stop(frame) @@ -494,28 +660,35 @@ class SpeechmaticsSTTService(STTService): # CONFIGURATION # ============================================================================ - def _prepare_config(self, params: InputParams) -> VoiceAgentConfig: - """Parse the InputParams into VoiceAgentConfig.""" - # Preset - config = VoiceAgentConfigPreset.load(params.turn_detection_mode.value) + def _build_config(self) -> VoiceAgentConfig: + """Build a ``VoiceAgentConfig`` from the current ``self._settings``. + + Used both at init time and before reconnecting so the connection + always reflects the latest settings. + """ + s = self._settings + + # Preset from turn detection mode + config = VoiceAgentConfigPreset.load(s.turn_detection_mode.value) # Language + domain - config.language = self._language_to_speechmatics_language(params.language) - config.domain = params.domain - config.output_locale = self._locale_to_speechmatics_locale(config.language, params.language) + language = s.language + config.language = self._language_to_speechmatics_language(language) + config.domain = s.domain if is_given(s.domain) else None + config.output_locale = self._locale_to_speechmatics_locale(config.language, language) # Speaker config config.speaker_config = SpeakerFocusConfig( - focus_speakers=params.focus_speakers, - ignore_speakers=params.ignore_speakers, - focus_mode=params.focus_mode, + focus_speakers=s.focus_speakers if is_given(s.focus_speakers) else [], + ignore_speakers=s.ignore_speakers if is_given(s.ignore_speakers) else [], + focus_mode=s.focus_mode if is_given(s.focus_mode) else SpeakerFocusMode.RETAIN, ) - config.known_speakers = params.known_speakers + config.known_speakers = s.known_speakers if is_given(s.known_speakers) else [] # Custom dictionary - config.additional_vocab = params.additional_vocab + config.additional_vocab = s.additional_vocab if is_given(s.additional_vocab) else [] - # Advanced parameters + # Advanced parameters — only set if given (not NOT_GIVEN or None) for param in [ "operating_point", "max_delay", @@ -529,21 +702,20 @@ class SpeechmaticsSTTService(STTService): "max_speakers", "prefer_current_speaker", ]: - if getattr(params, param) is not None: - setattr(config, param, getattr(params, param)) + val = getattr(s, param) + if is_given(val) and val is not None: + setattr(config, param, val) # Extra parameters - if isinstance(params.extra_params, dict): - for key, value in params.extra_params.items(): + if is_given(s.extra_params) and isinstance(s.extra_params, dict): + for key, value in s.extra_params.items(): if hasattr(config, key): setattr(config, key, value) # Enable sentences - config.speech_segment_config = SpeechSegmentConfig( - emit_sentences=params.split_sentences or False - ) + split = s.split_sentences if is_given(s.split_sentences) else False + config.speech_segment_config = SpeechSegmentConfig(emit_sentences=split or False) - # Return the complete config return config def update_params( @@ -552,12 +724,23 @@ class SpeechmaticsSTTService(STTService): ) -> None: """Updates the speaker configuration. + .. deprecated:: + Use ``STTUpdateSettingsFrame`` with + ``SpeechmaticsSTTSettings(...)`` instead. + This can update the speakers to listen to or ignore during an in-flight transcription. Only available if diarization is enabled. Args: params: Update parameters for the service. """ + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "update_params() is deprecated. Use STTUpdateSettingsFrame with " + "SpeechmaticsSTTSettings(...) instead.", + DeprecationWarning, + ) # Check possible if not self._config.enable_diarization: raise ValueError("Diarization is not enabled") @@ -727,9 +910,9 @@ class SpeechmaticsSTTService(STTService): def attr_from_segment(segment: dict[str, Any]) -> dict[str, Any]: # Formats the output text based on the speaker and defined formats from the config. text = ( - self._speaker_active_format + self._settings.speaker_active_format if segment.get("is_active", True) - else self._speaker_passive_format + else self._settings.speaker_passive_format ).format( **{ "speaker_id": segment.get("speaker_id", "UU"), diff --git a/src/pipecat/services/speechmatics/tts.py b/src/pipecat/services/speechmatics/tts.py index 0f3ff0cb6..7c8d9fca5 100644 --- a/src/pipecat/services/speechmatics/tts.py +++ b/src/pipecat/services/speechmatics/tts.py @@ -7,7 +7,8 @@ """Speechmatics TTS service integration.""" import asyncio -from typing import AsyncGenerator, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, Optional from urllib.parse import urlencode import aiohttp @@ -21,6 +22,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven from pipecat.services.tts_service import TTSService from pipecat.utils.network import exponential_backoff_time from pipecat.utils.tracing.service_decorators import traced_tts @@ -35,6 +37,17 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class SpeechmaticsTTSSettings(TTSSettings): + """Settings for Speechmatics TTS service. + + Parameters: + max_retries: Maximum number of retries for HTTP requests. + """ + + max_retries: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class SpeechmaticsTTSService(TTSService): """Speechmatics TTS service implementation. @@ -42,6 +55,8 @@ class SpeechmaticsTTSService(TTSService): It converts text to speech and returns raw PCM audio data for real-time playback. """ + _settings: SpeechmaticsTTSSettings + SPEECHMATICS_SAMPLE_RATE = 16000 class InputParams(BaseModel): @@ -91,11 +106,11 @@ class SpeechmaticsTTSService(TTSService): if not self._api_key: raise ValueError("Missing Speechmatics API key") - # Default parameters - self._params = params or SpeechmaticsTTSService.InputParams() - - # Set voice from constructor parameter - self.set_voice(voice_id) + params = params or SpeechmaticsTTSService.InputParams() + self._settings = SpeechmaticsTTSSettings( + voice=voice_id, + max_retries=params.max_retries, + ) def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -131,7 +146,7 @@ class SpeechmaticsTTSService(TTSService): } # Complete HTTP URL - url = _get_endpoint_url(self._base_url, self._voice_id, self.sample_rate) + url = _get_endpoint_url(self._base_url, self._settings.voice, self.sample_rate) try: # Start TTS TTFB metrics @@ -159,7 +174,7 @@ class SpeechmaticsTTSService(TTSService): attempt += 1 # Check if we've exceeded the maximum number of attempts - if attempt >= self._params.max_retries: + if attempt >= self._settings.max_retries: raise ValueError() # Report error frame diff --git a/src/pipecat/services/stt_service.py b/src/pipecat/services/stt_service.py index bad8e2e28..dfdae6de6 100644 --- a/src/pipecat/services/stt_service.py +++ b/src/pipecat/services/stt_service.py @@ -9,9 +9,10 @@ import asyncio import io import time +import warnings import wave from abc import abstractmethod -from typing import Any, AsyncGenerator, Dict, Mapping, Optional +from typing import Any, AsyncGenerator, Optional from loguru import logger from websockets.protocol import State @@ -32,6 +33,7 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_service import AIService +from pipecat.services.settings import STTSettings, is_given from pipecat.services.stt_latency import DEFAULT_TTFS_P99 from pipecat.services.websocket_service import WebsocketService from pipecat.transcriptions.language import Language @@ -73,6 +75,8 @@ class STTService(AIService): logger.error(f"STT connection error: {error}") """ + _settings: STTSettings + def __init__( self, *, @@ -111,7 +115,8 @@ class STTService(AIService): self._audio_passthrough = audio_passthrough self._init_sample_rate = sample_rate self._sample_rate = 0 - self._settings: Dict[str, Any] = {} + + self._settings = STTSettings() # Here in case subclass doesn't implement more specific settings (hopefully shouldn't happen) self._muted: bool = False self._user_id: str = "" self._ttfs_p99_latency = ttfs_p99_latency @@ -179,18 +184,53 @@ class STTService(AIService): async def set_model(self, model: str): """Set the speech recognition model. + .. deprecated:: 0.0.103 + Use ``STTUpdateSettingsFrame(model=...)`` instead. + Args: model: The name of the model to use for speech recognition. """ - self.set_model_name(model) + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "'set_model' is deprecated, use 'STTUpdateSettingsFrame(model=...)' instead.", + DeprecationWarning, + stacklevel=2, + ) + logger.info(f"Switching STT model to: [{model}]") + settings_cls = type(self._settings) + await self._update_settings(settings_cls(model=model)) async def set_language(self, language: Language): """Set the language for speech recognition. + .. deprecated:: 0.0.103 + Use ``STTUpdateSettingsFrame(language=...)`` instead. + Args: language: The language to use for speech recognition. """ - pass + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "'set_language' is deprecated, use 'STTUpdateSettingsFrame(language=...)' instead.", + DeprecationWarning, + stacklevel=2, + ) + logger.info(f"Switching STT language to: [{language}]") + settings_cls = type(self._settings) + await self._update_settings(settings_cls(language=language)) + + def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a language to the service-specific language format. + + Args: + language: The language to convert. + + Returns: + The service-specific language identifier, or None if not supported. + """ + return Language(language) @abstractmethod async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: @@ -222,20 +262,29 @@ class STTService(AIService): await self._cancel_ttfb_timeout() await self._cancel_keepalive_task() - async def _update_settings(self, settings: Mapping[str, Any]): - logger.info(f"Updating STT settings: {self._settings}") - for key, value in settings.items(): - if key in self._settings: - logger.info(f"Updating STT setting {key} to: [{value}]") - self._settings[key] = value - if key == "language": - await self.set_language(value) - elif key == "language": - await self.set_language(value) - elif key == "model": - self.set_model_name(value) - else: - logger.warning(f"Unknown setting for STT service: {key}") + async def _update_settings(self, update: STTSettings) -> dict[str, Any]: + """Apply an STT settings update. + + Handles ``model`` (via parent). Translates ``Language`` enum values + before applying so the stored value is a service-specific string. + Concrete services should override this method and handle language + changes (including any reconnect logic) based on the returned + changed-field dict. + + Args: + update: An STT settings delta. + + Returns: + Dict mapping changed field names to their previous values. + """ + # Translate language *before* applying so the stored value is canonical + if is_given(update.language) and isinstance(update.language, Language): + converted = self.language_to_service_language(update.language) + if converted is not None: + update.language = converted + + changed = await super()._update_settings(update) + return changed async def process_audio_frame(self, frame: AudioRawFrame, direction: FrameDirection): """Process an audio frame for speech recognition. @@ -300,7 +349,20 @@ class STTService(AIService): await self._handle_vad_user_stopped_speaking(frame) await self.push_frame(frame, direction) elif isinstance(frame, STTUpdateSettingsFrame): - await self._update_settings(frame.settings) + if frame.update is not None: + await self._update_settings(frame.update) + elif frame.settings: + # Backward-compatible path: convert legacy dict to settings object. + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Passing a dict via STTUpdateSettingsFrame(settings={...}) is deprecated " + "since 0.0.103, use STTUpdateSettingsFrame(update=STTSettings(...)) instead.", + DeprecationWarning, + stacklevel=2, + ) + update = type(self._settings).from_mapping(frame.settings) + await self._update_settings(update) elif isinstance(frame, STTMuteFrame): self._muted = frame.mute logger.debug(f"STT service {'muted' if frame.mute else 'unmuted'}") diff --git a/src/pipecat/services/tts_service.py b/src/pipecat/services/tts_service.py index 1e5bdf73f..9d9c41b60 100644 --- a/src/pipecat/services/tts_service.py +++ b/src/pipecat/services/tts_service.py @@ -8,6 +8,7 @@ import asyncio import uuid +import warnings from abc import abstractmethod from dataclasses import dataclass from typing import ( @@ -18,7 +19,6 @@ from typing import ( Callable, Dict, List, - Mapping, Optional, Sequence, Tuple, @@ -52,6 +52,7 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_service import AIService +from pipecat.services.settings import TTSSettings, is_given from pipecat.services.websocket_service import WebsocketService from pipecat.transcriptions.language import Language from pipecat.utils.text.base_text_aggregator import BaseTextAggregator @@ -103,6 +104,8 @@ class TTSService(AIService): logger.debug(f"TTS request: {context_id} - {text}") """ + _settings: TTSSettings + def __init__( self, *, @@ -188,8 +191,9 @@ class TTSService(AIService): self._append_trailing_space: bool = append_trailing_space self._init_sample_rate = sample_rate self._sample_rate = 0 - self._voice_id: str = "" - self._settings: Dict[str, Any] = {} + self._settings = TTSSettings( + voice="" + ) # Here in case subclass doesn't implement more specific settings (hopefully shouldn't happen) self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator() if text_aggregator: import warnings @@ -261,18 +265,42 @@ class TTSService(AIService): async def set_model(self, model: str): """Set the TTS model to use. + .. deprecated:: 0.0.103 + Use ``TTSUpdateSettingsFrame(model=...)`` instead. + Args: model: The name of the TTS model. """ - self.set_model_name(model) + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "'set_model' is deprecated, use 'TTSUpdateSettingsFrame(model=...)' instead.", + DeprecationWarning, + stacklevel=2, + ) + logger.info(f"Switching TTS model to: [{model}]") + settings_cls = type(self._settings) + await self._update_settings(settings_cls(model=model)) - def set_voice(self, voice: str): + async def set_voice(self, voice: str): """Set the voice for speech synthesis. + .. deprecated:: 0.0.103 + Use ``TTSUpdateSettingsFrame(voice=...)`` instead. + Args: voice: The voice identifier or name. """ - self._voice_id = voice + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "'set_voice' is deprecated, use 'TTSUpdateSettingsFrame(voice=...)' instead.", + DeprecationWarning, + stacklevel=2, + ) + logger.info(f"Switching TTS voice to: [{voice}]") + settings_cls = type(self._settings) + await self._update_settings(settings_cls(voice=voice)) def create_context_id(self) -> str: """Generate a unique context ID for a TTS request. @@ -324,15 +352,6 @@ class TTSService(AIService): return text + " " return text - async def update_setting(self, key: str, value: Any): - """Update a service-specific setting. - - Args: - key: The setting key to update. - value: The new value for the setting. - """ - pass - async def flush_audio(self): """Flush any buffered audio data.""" pass @@ -403,22 +422,26 @@ class TTSService(AIService): if not (agg_type == aggregation_type and func == transform_function) ] - async def _update_settings(self, settings: Mapping[str, Any]): - for key, value in settings.items(): - if key in self._settings: - logger.info(f"Updating TTS setting {key} to: [{value}]") - self._settings[key] = value - if key == "language": - self._settings[key] = self.language_to_service_language(value) - elif key == "model": - self.set_model_name(value) - elif key == "voice" or key == "voice_id": - self.set_voice(value) - elif key == "text_filter": - for filter in self._text_filters: - await filter.update_settings(value) - else: - logger.warning(f"Unknown setting for TTS service: {key}") + async def _update_settings(self, update: TTSSettings) -> dict[str, Any]: + """Apply a TTS settings update. + + Translates language to service-specific value before applying. + + Args: + update: A TTS settings delta. + + Returns: + Dict mapping changed field names to their previous values. + """ + # Translate language *before* applying so the stored value is canonical + if is_given(update.language) and isinstance(update.language, Language): + converted = self.language_to_service_language(update.language) + if converted is not None: + update.language = converted + + changed = await super()._update_settings(update) + + return changed async def say(self, text: str): """Immediately speak the provided text. @@ -501,7 +524,20 @@ class TTSService(AIService): await self.flush_audio() self._processing_text = processing_text elif isinstance(frame, TTSUpdateSettingsFrame): - await self._update_settings(frame.settings) + if frame.update is not None: + await self._update_settings(frame.update) + elif frame.settings: + # Backward-compatible path: convert legacy dict to settings object. + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Passing a dict via TTSUpdateSettingsFrame(settings={...}) is deprecated " + "since 0.0.103, use TTSUpdateSettingsFrame(update=TTSSettings(...)) instead.", + DeprecationWarning, + stacklevel=2, + ) + update = type(self._settings).from_mapping(frame.settings) + await self._update_settings(update) elif isinstance(frame, BotStoppedSpeakingFrame): await self._maybe_resume_frame_processing() await self.push_frame(frame, direction) diff --git a/src/pipecat/services/ultravox/llm.py b/src/pipecat/services/ultravox/llm.py index d549b11e5..436653c7e 100644 --- a/src/pipecat/services/ultravox/llm.py +++ b/src/pipecat/services/ultravox/llm.py @@ -15,6 +15,7 @@ import asyncio import datetime import json import uuid +from dataclasses import dataclass, field from typing import Any, Dict, List, Literal, Optional, Union import aiohttp @@ -34,7 +35,6 @@ from pipecat.frames.frames import ( LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMTextFrame, - LLMUpdateSettingsFrame, StartFrame, TranscriptionFrame, TTSAudioRawFrame, @@ -56,6 +56,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import FunctionCallFromLLM, LLMService +from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven from pipecat.utils.time import time_now_iso8601 try: @@ -66,6 +67,17 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class UltravoxRealtimeLLMSettings(LLMSettings): + """Settings for UltravoxRealtimeLLMService. + + Parameters: + output_medium: The output medium for the model ("voice" or "text"). + """ + + output_medium: str | _NotGiven = field(default=NOT_GIVEN) + + class AgentInputParams(BaseModel): """Input parameters for Ultravox Realtime generation using a pre-defined Agent. @@ -146,6 +158,8 @@ class UltravoxRealtimeLLMService(LLMService): by the model and may not always align with its understanding of user input. """ + _settings: UltravoxRealtimeLLMSettings + def __init__( self, *, @@ -163,6 +177,7 @@ class UltravoxRealtimeLLMService(LLMService): **kwargs: Additional arguments passed to parent LLMService. """ super().__init__(**kwargs) + self._settings = UltravoxRealtimeLLMSettings() self._params = params if one_shot_selected_tools: if not isinstance(self._params, OneShotInputParams): @@ -310,6 +325,13 @@ class UltravoxRealtimeLLMService(LLMService): await self.cancel_task(self._receive_task, timeout=1.0) self._receive_task = None + async def _update_settings(self, update: UltravoxRealtimeLLMSettings): + changed = await super()._update_settings(update) + if "output_medium" in changed: + await self._update_output_medium(self._settings.output_medium) + self._warn_unhandled_updated_settings(changed.keys() - {"output_medium"}) + return changed + # # frame processing # StartFrame, StopFrame, CancelFrame implemented in base class @@ -331,9 +353,6 @@ class UltravoxRealtimeLLMService(LLMService): else LLMContext.from_openai_context(frame.context) ) await self._handle_context(context) - elif isinstance(frame, LLMUpdateSettingsFrame): - if "output_medium" in frame.settings: - await self._update_output_medium(frame.settings.get("output_medium")) elif isinstance(frame, InputTextRawFrame): await self._send_user_text(frame.text) await self.push_frame(frame, direction) diff --git a/src/pipecat/services/whisper/base_stt.py b/src/pipecat/services/whisper/base_stt.py index bc999dba4..9def3c2f1 100644 --- a/src/pipecat/services/whisper/base_stt.py +++ b/src/pipecat/services/whisper/base_stt.py @@ -10,13 +10,15 @@ This module provides common functionality for services implementing the Whisper interface, including language mapping, metrics generation, and error handling. """ -from typing import AsyncGenerator, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, Optional from loguru import logger from openai import AsyncOpenAI from openai.types.audio import Transcription from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame +from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven from pipecat.services.stt_latency import WHISPER_TTFS_P99 from pipecat.services.stt_service import SegmentedSTTService from pipecat.transcriptions.language import Language, resolve_language @@ -24,6 +26,22 @@ from pipecat.utils.time import time_now_iso8601 from pipecat.utils.tracing.service_decorators import traced_stt +@dataclass +class BaseWhisperSTTSettings(STTSettings): + """Settings for Whisper API-based STT services. + + Parameters: + base_url: API base URL. + prompt: Optional text to guide the model's style or continue + a previous segment. + temperature: Sampling temperature between 0 and 1. + """ + + base_url: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + prompt: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + temperature: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + def language_to_whisper_language(language: Language) -> Optional[str]: """Maps pipecat Language enum to Whisper API language codes. @@ -106,6 +124,8 @@ class BaseWhisperSTTService(SegmentedSTTService): including metrics generation and error handling. """ + _settings: BaseWhisperSTTSettings + def __init__( self, *, @@ -136,33 +156,43 @@ class BaseWhisperSTTService(SegmentedSTTService): **kwargs: Additional arguments passed to SegmentedSTTService. """ super().__init__(ttfs_p99_latency=ttfs_p99_latency, **kwargs) - self.set_model_name(model) self._client = self._create_client(api_key, base_url) self._language = self.language_to_service_language(language or Language.EN) self._prompt = prompt self._temperature = temperature self._include_prob_metrics = include_prob_metrics - self._settings = { - "base_url": base_url, - "language": self._language, - "prompt": self._prompt, - "temperature": self._temperature, - } + self._settings = BaseWhisperSTTSettings( + model=model, + language=self._language, + base_url=base_url, + prompt=self._prompt, + temperature=self._temperature, + ) + self._sync_model_name_to_metrics() def _create_client(self, api_key: Optional[str], base_url: Optional[str]): return AsyncOpenAI(api_key=api_key, base_url=base_url) - async def set_model(self, model: str): - """Set the model name for transcription. + async def _update_settings(self, update: STTSettings) -> dict[str, Any]: + """Apply a settings update, syncing instance variables. - Args: - model: The name of the model to use. + Keeps ``_language``, ``_prompt``, and ``_temperature`` in sync with + the settings fields. """ - self.set_model_name(model) + changed = await super()._update_settings(update) + + if "language" in changed: + self._language = self._settings.language + if "prompt" in changed: + self._prompt = self._settings.prompt + if "temperature" in changed: + self._temperature = self._settings.temperature + + return changed def can_generate_metrics(self) -> bool: - """Indicates whether this service can generate metrics. + """Whether this service can generate processing metrics. Returns: bool: True, as this service supports metric generation. @@ -180,15 +210,6 @@ class BaseWhisperSTTService(SegmentedSTTService): """ return language_to_whisper_language(language) - async def set_language(self, language: Language): - """Set the language for transcription. - - Args: - language: The Language enum value to use for transcription. - """ - logger.info(f"Switching STT language to: [{language}]") - self._language = self.language_to_service_language(language) - @traced_stt async def _handle_transcription( self, transcript: str, is_final: bool, language: Optional[Language] = None diff --git a/src/pipecat/services/whisper/stt.py b/src/pipecat/services/whisper/stt.py index f11978cc2..205838314 100644 --- a/src/pipecat/services/whisper/stt.py +++ b/src/pipecat/services/whisper/stt.py @@ -11,6 +11,7 @@ supporting both Faster Whisper and MLX Whisper backends for efficient inference. """ import asyncio +from dataclasses import dataclass, field from enum import Enum from typing import AsyncGenerator, Optional @@ -19,6 +20,7 @@ from loguru import logger from typing_extensions import TYPE_CHECKING, override from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame +from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven from pipecat.services.stt_service import SegmentedSTTService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.time import time_now_iso8601 @@ -172,6 +174,36 @@ def language_to_whisper_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +@dataclass +class WhisperSTTSettings(STTSettings): + """Settings for the local Whisper (Faster Whisper) STT service. + + Parameters: + device: Inference device ('cpu', 'cuda', or 'auto'). + compute_type: Compute type for inference ('default', 'int8', etc.). + no_speech_prob: Probability threshold for filtering non-speech segments. + """ + + device: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + compute_type: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + no_speech_prob: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + +@dataclass +class WhisperMLXSTTSettings(STTSettings): + """Settings for the MLX Whisper STT service. + + Parameters: + no_speech_prob: Probability threshold for filtering non-speech segments. + temperature: Sampling temperature (0.0-1.0). + engine: Whisper engine identifier. + """ + + no_speech_prob: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + temperature: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + engine: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class WhisperSTTService(SegmentedSTTService): """Class to transcribe audio with a locally-downloaded Whisper model. @@ -179,6 +211,8 @@ class WhisperSTTService(SegmentedSTTService): segments. It supports multiple languages and various model sizes. """ + _settings: WhisperSTTSettings + def __init__( self, *, @@ -202,16 +236,17 @@ class WhisperSTTService(SegmentedSTTService): super().__init__(**kwargs) self._device: str = device self._compute_type = compute_type - self.set_model_name(model if isinstance(model, str) else model.value) self._no_speech_prob = no_speech_prob self._model: Optional[WhisperModel] = None - self._settings = { - "language": language, - "device": self._device, - "compute_type": self._compute_type, - "no_speech_prob": self._no_speech_prob, - } + self._settings = WhisperSTTSettings( + model=model if isinstance(model, str) else model.value, + language=language, + device=self._device, + compute_type=self._compute_type, + no_speech_prob=self._no_speech_prob, + ) + self._sync_model_name_to_metrics() self._load() @@ -234,15 +269,6 @@ class WhisperSTTService(SegmentedSTTService): """ return language_to_whisper_language(language) - async def set_language(self, language: Language): - """Set the language for transcription. - - Args: - language: The Language enum value to use for transcription. - """ - logger.info(f"Switching STT language to: [{language}]") - self._settings["language"] = language - def _load(self): """Loads the Whisper model. @@ -255,7 +281,7 @@ class WhisperSTTService(SegmentedSTTService): logger.debug("Loading Whisper model...") self._model = WhisperModel( - self.model_name, device=self._device, compute_type=self._compute_type + self._settings.model, device=self._device, compute_type=self._compute_type ) logger.debug("Loaded Whisper model") except ModuleNotFoundError as e: @@ -293,9 +319,8 @@ class WhisperSTTService(SegmentedSTTService): # Divide by 32768 because we have signed 16-bit data. audio_float = np.frombuffer(audio, dtype=np.int16).astype(np.float32) / 32768.0 - whisper_lang = self.language_to_service_language(self._settings["language"]) segments, _ = await asyncio.to_thread( - self._model.transcribe, audio_float, language=whisper_lang + self._model.transcribe, audio_float, language=self._settings.language ) text: str = "" for segment in segments: @@ -305,13 +330,13 @@ class WhisperSTTService(SegmentedSTTService): await self.stop_processing_metrics() if text: - await self._handle_transcription(text, True, self._settings["language"]) + await self._handle_transcription(text, True, self._settings.language) logger.debug(f"Transcription: [{text}]") yield TranscriptionFrame( text, self._user_id, time_now_iso8601(), - self._settings["language"], + self._settings.language, ) @@ -322,6 +347,8 @@ class WhisperSTTServiceMLX(WhisperSTTService): segments. It's optimized for Apple Silicon and supports multiple languages and quantizations. """ + _settings: WhisperMLXSTTSettings + def __init__( self, *, @@ -343,16 +370,17 @@ class WhisperSTTServiceMLX(WhisperSTTService): # Skip WhisperSTTService.__init__ and call its parent directly SegmentedSTTService.__init__(self, **kwargs) - self.set_model_name(model if isinstance(model, str) else model.value) self._no_speech_prob = no_speech_prob self._temperature = temperature - self._settings = { - "language": language, - "no_speech_prob": self._no_speech_prob, - "temperature": self._temperature, - "engine": "mlx", - } + self._settings = WhisperMLXSTTSettings( + model=model if isinstance(model, str) else model.value, + language=language, + no_speech_prob=self._no_speech_prob, + temperature=self._temperature, + engine="mlx", + ) + self._sync_model_name_to_metrics() # No need to call _load() as MLX Whisper loads models on demand @@ -390,13 +418,12 @@ class WhisperSTTServiceMLX(WhisperSTTService): # Divide by 32768 because we have signed 16-bit data. audio_float = np.frombuffer(audio, dtype=np.int16).astype(np.float32) / 32768.0 - whisper_lang = self.language_to_service_language(self._settings["language"]) chunk = await asyncio.to_thread( mlx_whisper.transcribe, audio_float, - path_or_hf_repo=self.model_name, + path_or_hf_repo=self._settings.model, temperature=self._temperature, - language=whisper_lang, + language=self._settings.language, ) text: str = "" for segment in chunk.get("segments", []): @@ -413,13 +440,13 @@ class WhisperSTTServiceMLX(WhisperSTTService): await self.stop_processing_metrics() if text: - await self._handle_transcription(text, True, self._settings["language"]) + await self._handle_transcription(text, True, self._settings.language) logger.debug(f"Transcription: [{text}]") yield TranscriptionFrame( text, self._user_id, time_now_iso8601(), - self._settings["language"], + self._settings.language, ) except Exception as e: diff --git a/src/pipecat/services/xtts/tts.py b/src/pipecat/services/xtts/tts.py index bf4eb4f03..ba2eb4fc2 100644 --- a/src/pipecat/services/xtts/tts.py +++ b/src/pipecat/services/xtts/tts.py @@ -10,7 +10,8 @@ This module provides integration with Coqui XTTS streaming server for text-to-speech synthesis using local Docker deployment. """ -from typing import Any, AsyncGenerator, Dict, Optional +from dataclasses import dataclass, field +from typing import AsyncGenerator, Dict, Optional import aiohttp from loguru import logger @@ -24,6 +25,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven from pipecat.services.tts_service import TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -68,6 +70,17 @@ def language_to_xtts_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +@dataclass +class XTTSTTSSettings(TTSSettings): + """Settings for XTTS TTS service. + + Parameters: + base_url: Base URL of the XTTS streaming server. + """ + + base_url: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + class XTTSService(TTSService): """Coqui XTTS text-to-speech service. @@ -76,6 +89,8 @@ class XTTSService(TTSService): studio speakers configuration. """ + _settings: XTTSTTSSettings + def __init__( self, *, @@ -98,11 +113,11 @@ class XTTSService(TTSService): """ super().__init__(sample_rate=sample_rate, **kwargs) - self._settings = { - "language": self.language_to_service_language(language), - "base_url": base_url, - } - self.set_voice(voice_id) + self._settings = XTTSTTSSettings( + voice=voice_id, + language=self.language_to_service_language(language), + base_url=base_url, + ) self._studio_speakers: Optional[Dict[str, Any]] = None self._aiohttp_session = aiohttp_session @@ -138,7 +153,7 @@ class XTTSService(TTSService): if self._studio_speakers: return - async with self._aiohttp_session.get(self._settings["base_url"] + "/studio_speakers") as r: + async with self._aiohttp_session.get(self._settings.base_url + "/studio_speakers") as r: if r.status != 200: text = await r.text() await self.push_error( @@ -164,13 +179,13 @@ class XTTSService(TTSService): logger.error(f"{self} no studio speakers available") return - embeddings = self._studio_speakers[self._voice_id] + embeddings = self._studio_speakers[self._settings.voice] - url = self._settings["base_url"] + "/tts_stream" + url = self._settings.base_url + "/tts_stream" payload = { "text": text.replace(".", "").replace("*", ""), - "language": self._settings["language"], + "language": self._settings.language, "speaker_embedding": embeddings["speaker_embedding"], "gpt_cond_latent": embeddings["gpt_cond_latent"], "add_wav_header": False, diff --git a/src/pipecat/utils/tracing/service_decorators.py b/src/pipecat/utils/tracing/service_decorators.py index 968fe8e8a..601cad53d 100644 --- a/src/pipecat/utils/tracing/service_decorators.py +++ b/src/pipecat/utils/tracing/service_decorators.py @@ -42,6 +42,23 @@ T = TypeVar("T") R = TypeVar("R") +def _get_model_name(service) -> str: + """Get the model name from a service instance. + + This is a bit of a mess — there were multiple places a model name could live. + Soon, self._settings should be the only source of truth about model name. + In fact...it might already be the case, but juuuuust to be safe, we'll + check all the places we used to store it. + """ + return ( + getattr(getattr(service, "_settings", None), "model", None) + or getattr(service, "_full_model_name", None) + or getattr(service, "model_name", None) + or getattr(service, "_model_name", None) + or "unknown" + ) + + def _noop_decorator(func): """No-op fallback decorator when tracing is unavailable. @@ -202,13 +219,14 @@ def traced_tts(func: Optional[Callable] = None, *, name: Optional[str] = None) - tracer = trace.get_tracer("pipecat") with tracer.start_as_current_span(span_name, context=parent_context) as span: try: + settings = getattr(self, "_settings", {}) add_tts_span_attributes( span=span, service_name=service_class_name, - model=getattr(self, "model_name") or "unknown", - voice_id=getattr(self, "_voice_id", "unknown"), + model=_get_model_name(self), + voice_id=getattr(settings, "voice", "unknown"), text=text, - settings=getattr(self, "_settings", {}), + settings=settings, character_count=len(text), operation_name="tts", cartesia_version=getattr(self, "_cartesia_version", None), @@ -325,7 +343,7 @@ def traced_stt(func: Optional[Callable] = None, *, name: Optional[str] = None) - add_stt_span_attributes( span=current_span, service_name=service_class_name, - model=getattr(self, "model_name") or settings.get("model", "unknown"), + model=_get_model_name(self), transcript=transcript, is_final=is_final, language=str(language) if language else None, @@ -506,10 +524,7 @@ def traced_llm(func: Optional[Callable] = None, *, name: Optional[str] = None) - # Add all available attributes to the span attribute_kwargs = { "service_name": service_class_name, - "model": getattr(self, "_full_model_name", None) - or getattr(self, "model_name", None) - or params.get("model") - or "unknown", + "model": _get_model_name(self), "stream": True, # Most LLM services use streaming "parameters": params, } @@ -609,11 +624,7 @@ def traced_gemini_live(operation: str) -> Callable: ) as current_span: try: # Base service attributes - model_name = ( - getattr(self, "model_name", None) - or getattr(self, "_model_name", None) - or "unknown" - ) + model_name = _get_model_name(self) voice_id = getattr(self, "_voice_id", None) language_code = getattr(self, "_language_code", None) settings = getattr(self, "_settings", {}) @@ -917,11 +928,7 @@ def traced_openai_realtime(operation: str) -> Callable: ) as current_span: try: # Base service attributes - model_name = ( - getattr(self, "model_name", None) - or getattr(self, "_model_name", None) - or "unknown" - ) + model_name = _get_model_name(self) # Operation-specific attribute collection operation_attrs = {} diff --git a/tests/test_settings.py b/tests/test_settings.py new file mode 100644 index 000000000..85f89987c --- /dev/null +++ b/tests/test_settings.py @@ -0,0 +1,313 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Tests for the typed settings infrastructure in pipecat.services.settings.""" + +import pytest + +from pipecat.services.settings import ( + NOT_GIVEN, + LLMSettings, + ServiceSettings, + STTSettings, + TTSSettings, + _NotGiven, + is_given, +) + +# --------------------------------------------------------------------------- +# NOT_GIVEN sentinel +# --------------------------------------------------------------------------- + + +class TestNotGiven: + def test_singleton(self): + """NOT_GIVEN is a singleton — every reference is the same object.""" + assert _NotGiven() is _NotGiven() + assert NOT_GIVEN is _NotGiven() + + def test_repr(self): + assert repr(NOT_GIVEN) == "NOT_GIVEN" + + def test_bool_is_false(self): + assert not NOT_GIVEN + assert bool(NOT_GIVEN) is False + + def test_is_given_with_not_given(self): + assert is_given(NOT_GIVEN) is False + + def test_is_given_with_none(self): + assert is_given(None) is True + + def test_is_given_with_values(self): + assert is_given(0) is True + assert is_given("") is True + assert is_given(False) is True + assert is_given(42) is True + assert is_given("hello") is True + + +# --------------------------------------------------------------------------- +# ServiceSettings base +# --------------------------------------------------------------------------- + + +class TestServiceSettings: + def test_default_fields_are_not_given(self): + s = ServiceSettings() + assert not is_given(s.model) + assert s.extra == {} + + def test_given_fields_empty_by_default(self): + s = ServiceSettings() + assert s.given_fields() == {} + + def test_given_fields_includes_set_values(self): + s = ServiceSettings(model="gpt-4o") + assert s.given_fields() == {"model": "gpt-4o"} + + def test_given_fields_includes_extra(self): + s = ServiceSettings(model="gpt-4o") + s.extra = {"custom_key": 42} + result = s.given_fields() + assert result == {"model": "gpt-4o", "custom_key": 42} + + def test_copy_is_deep(self): + s = ServiceSettings(model="gpt-4o") + s.extra = {"nested": {"a": 1}} + c = s.copy() + assert c.model == "gpt-4o" + assert c.extra == {"nested": {"a": 1}} + # Mutating the copy shouldn't affect the original + c.extra["nested"]["a"] = 999 + assert s.extra["nested"]["a"] == 1 + + +# --------------------------------------------------------------------------- +# apply_update +# --------------------------------------------------------------------------- + + +class TestApplyUpdate: + def test_apply_update_basic(self): + current = TTSSettings(voice="alice", language="en") + delta = TTSSettings(voice="bob") + changed = current.apply_update(delta) + assert changed.keys() == {"voice"} + assert changed["voice"] == "alice" # old value + assert current.voice == "bob" + assert current.language == "en" + + def test_apply_update_no_change(self): + current = TTSSettings(voice="alice", language="en") + delta = TTSSettings(voice="alice") + changed = current.apply_update(delta) + assert changed == {} + assert current.voice == "alice" + + def test_apply_update_not_given_skipped(self): + current = TTSSettings(voice="alice", language="en") + delta = TTSSettings() # all NOT_GIVEN + changed = current.apply_update(delta) + assert changed == {} + assert current.voice == "alice" + assert current.language == "en" + + def test_apply_update_multiple_fields(self): + current = LLMSettings(temperature=0.7, max_tokens=100) + delta = LLMSettings(temperature=0.9, max_tokens=200, top_p=0.95) + changed = current.apply_update(delta) + assert changed.keys() == {"temperature", "max_tokens", "top_p"} + assert changed["temperature"] == 0.7 + assert changed["max_tokens"] == 100 + assert current.temperature == 0.9 + assert current.max_tokens == 200 + assert current.top_p == 0.95 + + def test_apply_update_extra_merged(self): + current = TTSSettings(voice="alice") + current.extra = {"speed": 1.0, "stability": 0.5} + delta = TTSSettings() + delta.extra = {"speed": 1.2} + changed = current.apply_update(delta) + assert "speed" in changed + assert changed["speed"] == 1.0 # old value + assert current.extra == {"speed": 1.2, "stability": 0.5} + + def test_apply_update_extra_no_change(self): + current = TTSSettings(voice="alice") + current.extra = {"speed": 1.0} + delta = TTSSettings() + delta.extra = {"speed": 1.0} + changed = current.apply_update(delta) + assert changed == {} + + def test_apply_update_model_field(self): + current = ServiceSettings(model="old-model") + delta = ServiceSettings(model="new-model") + changed = current.apply_update(delta) + assert changed.keys() == {"model"} + assert changed["model"] == "old-model" + assert current.model == "new-model" + + def test_apply_update_none_is_a_valid_value(self): + """Setting a field to None should be treated as a change from NOT_GIVEN.""" + current = TTSSettings() + delta = TTSSettings(language=None) + changed = current.apply_update(delta) + assert "language" in changed + assert current.language is None + + def test_apply_update_none_to_value(self): + current = TTSSettings(language=None) + delta = TTSSettings(language="en") + changed = current.apply_update(delta) + assert "language" in changed + assert changed["language"] is None # old value was None + assert current.language == "en" + + +# --------------------------------------------------------------------------- +# from_mapping +# --------------------------------------------------------------------------- + + +class TestFromMapping: + def test_basic_mapping(self): + s = TTSSettings.from_mapping({"voice": "alice", "language": "en"}) + assert s.voice == "alice" + assert s.language == "en" + assert not is_given(s.model) + + def test_alias_resolution(self): + """'voice_id' is an alias for 'voice' in TTSSettings.""" + s = TTSSettings.from_mapping({"voice_id": "alice"}) + assert s.voice == "alice" + + def test_unknown_keys_go_to_extra(self): + s = TTSSettings.from_mapping({"voice": "alice", "speed": 1.2, "stability": 0.5}) + assert s.voice == "alice" + assert s.extra == {"speed": 1.2, "stability": 0.5} + + def test_model_field(self): + s = LLMSettings.from_mapping({"model": "gpt-4o", "temperature": 0.7}) + assert s.model == "gpt-4o" + assert s.temperature == 0.7 + + def test_empty_mapping(self): + s = ServiceSettings.from_mapping({}) + assert s.given_fields() == {} + + def test_all_unknown_keys(self): + s = ServiceSettings.from_mapping({"foo": 1, "bar": 2}) + assert not is_given(s.model) + assert s.extra == {"foo": 1, "bar": 2} + + def test_llm_settings_from_mapping(self): + s = LLMSettings.from_mapping({"temperature": 0.5, "max_tokens": 1000, "custom_param": True}) + assert s.temperature == 0.5 + assert s.max_tokens == 1000 + assert s.extra == {"custom_param": True} + + def test_stt_settings_from_mapping(self): + s = STTSettings.from_mapping({"language": "fr", "model": "whisper-large"}) + assert s.language == "fr" + assert s.model == "whisper-large" + + +# --------------------------------------------------------------------------- +# LLMSettings specifics +# --------------------------------------------------------------------------- + + +class TestLLMSettings: + def test_all_fields_not_given_by_default(self): + s = LLMSettings() + for name in ( + "model", + "temperature", + "max_tokens", + "top_p", + "top_k", + "frequency_penalty", + "presence_penalty", + "seed", + ): + assert not is_given(getattr(s, name)), f"{name} should be NOT_GIVEN" + + def test_given_fields(self): + s = LLMSettings(temperature=0.7, seed=42) + assert s.given_fields() == {"temperature": 0.7, "seed": 42} + + +# --------------------------------------------------------------------------- +# TTSSettings specifics +# --------------------------------------------------------------------------- + + +class TestTTSSettings: + def test_all_fields_not_given_by_default(self): + s = TTSSettings() + for name in ("model", "voice", "language"): + assert not is_given(getattr(s, name)), f"{name} should be NOT_GIVEN" + + def test_aliases_class_var(self): + assert TTSSettings._aliases == {"voice_id": "voice"} + + def test_given_fields(self): + s = TTSSettings(voice="alice") + assert s.given_fields() == {"voice": "alice"} + + +# --------------------------------------------------------------------------- +# STTSettings specifics +# --------------------------------------------------------------------------- + + +class TestSTTSettings: + def test_all_fields_not_given_by_default(self): + s = STTSettings() + for name in ("model", "language"): + assert not is_given(getattr(s, name)), f"{name} should be NOT_GIVEN" + + def test_given_fields(self): + s = STTSettings(language="en", model="whisper-large") + assert s.given_fields() == {"language": "en", "model": "whisper-large"} + + +# --------------------------------------------------------------------------- +# Integration: roundtrip from_mapping → apply_update +# --------------------------------------------------------------------------- + + +class TestRoundtrip: + def test_from_mapping_then_apply_update(self): + """Simulate the real flow: dict arrives via frame, gets converted, applied.""" + # Simulating current service state + current = TTSSettings(model="eleven_turbo_v2_5", voice="alice", language="en") + current.extra = {"stability": 0.5, "speed": 1.0} + + # Incoming dict-based update + raw = {"voice_id": "bob", "speed": 1.2} + delta = TTSSettings.from_mapping(raw) + + changed = current.apply_update(delta) + assert changed.keys() == {"voice", "speed"} + assert changed["voice"] == "alice" + assert changed["speed"] == 1.0 + assert current.voice == "bob" + assert current.language == "en" + assert current.extra["speed"] == 1.2 + assert current.extra["stability"] == 0.5 + + def test_from_mapping_preserves_model(self): + current = LLMSettings(model="gpt-4o", temperature=0.7) + delta = LLMSettings.from_mapping({"model": "gpt-4o-mini", "temperature": 0.9}) + changed = current.apply_update(delta) + assert changed.keys() == {"model", "temperature"} + assert changed["model"] == "gpt-4o" + assert current.model == "gpt-4o-mini" + assert current.temperature == 0.9