From 9bd51cd88c7182246261486bb48087cbadf0a5b7 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 16 Apr 2026 18:04:42 -0400 Subject: [PATCH 01/49] Add incremental pyright type checking with CI enforcement Add pyrightconfig.json with basic type checking for zero-error modules (clocks, metrics, transcriptions, frames) and enforce via CI. The include list will expand as modules are fixed. --- .github/workflows/format.yaml | 4 ++++ pyrightconfig.json | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 pyrightconfig.json diff --git a/.github/workflows/format.yaml b/.github/workflows/format.yaml index 8284be043..c02245708 100644 --- a/.github/workflows/format.yaml +++ b/.github/workflows/format.yaml @@ -41,3 +41,7 @@ jobs: - name: Ruff linter (all rules) id: ruff-check run: uv run ruff check + + - name: Type check (pyright) + id: pyright + run: uv run pyright diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 000000000..fbca2b790 --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,16 @@ +{ + "typeCheckingMode": "basic", + "pythonVersion": "3.11", + "pythonPlatform": "All", + "include": [ + "src/pipecat/clocks", + "src/pipecat/metrics", + "src/pipecat/transcriptions", + "src/pipecat/frames" + ], + "exclude": [ + "**/*_pb2.py", + "**/__pycache__" + ], + "reportMissingImports": false +} From aa355e3d32e79a98ef8427ebacc505f8895a7a98 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 16 Apr 2026 18:25:10 -0400 Subject: [PATCH 02/49] Fix type errors in observers and add to pyright checked set Group three co-assigned fields (_start_frame_id, _start_frame_arrival_ns, _start_wall_clock) into a single _StartFrameInfo dataclass. This makes the "always set together" invariant structural rather than implicit, and fixes the incorrect str | None annotation on _start_frame_id (Frame.id is int). --- pyrightconfig.json | 3 +- .../observers/startup_timing_observer.py | 49 ++++++++++--------- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/pyrightconfig.json b/pyrightconfig.json index fbca2b790..69d2ba8db 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -6,7 +6,8 @@ "src/pipecat/clocks", "src/pipecat/metrics", "src/pipecat/transcriptions", - "src/pipecat/frames" + "src/pipecat/frames", + "src/pipecat/observers" ], "exclude": [ "**/*_pb2.py", diff --git a/src/pipecat/observers/startup_timing_observer.py b/src/pipecat/observers/startup_timing_observer.py index 6c6eb8204..ac63ab10a 100644 --- a/src/pipecat/observers/startup_timing_observer.py +++ b/src/pipecat/observers/startup_timing_observer.py @@ -49,6 +49,15 @@ from pipecat.processors.frame_processor import FrameProcessor _INTERNAL_TYPES = (PipelineSource, BasePipeline) +@dataclass +class _StartFrameInfo: + """Captured once when the first StartFrame arrives at a processor.""" + + frame_id: int + arrival_ns: int + wall_clock: float + + @dataclass class _ArrivalInfo: """Internal record of when a StartFrame arrived at a processor.""" @@ -175,8 +184,8 @@ class StartupTimingObserver(BaseObserver): # Collected timings in pipeline order. self._timings: list[ProcessorStartupTiming] = [] - # Lock onto the first StartFrame we see (by frame ID). - self._start_frame_id: str | None = None + # Captured once when the first StartFrame arrives. + self._start_frame: _StartFrameInfo | None = None # Whether we've already emitted the startup timing report. self._startup_timing_reported = False @@ -184,15 +193,9 @@ class StartupTimingObserver(BaseObserver): # Whether we've already measured transport timing. self._transport_timing_reported = False - # Timestamp (ns) when we first see a StartFrame arrive at a processor. - self._start_frame_arrival_ns: int | None = None - # Bot connected timing (stored for inclusion in the transport report). self._bot_connected_secs: float | None = None - # Wall clock time when the StartFrame was first seen. - self._start_wall_clock: float | None = None - self._register_event_handler("on_startup_timing_report") self._register_event_handler("on_transport_timing_report") @@ -233,11 +236,13 @@ class StartupTimingObserver(BaseObserver): return # Lock onto the first StartFrame. - if self._start_frame_id is None: - self._start_frame_id = data.frame.id - self._start_frame_arrival_ns = data.timestamp - self._start_wall_clock = time.time() - elif data.frame.id != self._start_frame_id: + if self._start_frame is None: + self._start_frame = _StartFrameInfo( + frame_id=data.frame.id, + arrival_ns=data.timestamp, + wall_clock=time.time(), + ) + elif data.frame.id != self._start_frame.frame_id: return if self._should_track(data.processor): @@ -268,16 +273,16 @@ class StartupTimingObserver(BaseObserver): if not isinstance(data.frame, StartFrame): return - if self._start_frame_id is not None and data.frame.id != self._start_frame_id: + if self._start_frame is not None and data.frame.id != self._start_frame.frame_id: return arrival = self._arrivals.pop(data.source.id, None) - if arrival is None: + if arrival is None or self._start_frame is None: return duration_ns = data.timestamp - arrival.arrival_ts_ns duration_secs = duration_ns / 1e9 - start_offset_secs = (arrival.arrival_ts_ns - self._start_frame_arrival_ns) / 1e9 + start_offset_secs = (arrival.arrival_ts_ns - self._start_frame.arrival_ns) / 1e9 self._timings.append( ProcessorStartupTiming( @@ -289,22 +294,22 @@ class StartupTimingObserver(BaseObserver): def _handle_bot_connected(self, data: FramePushed): """Record bot connected timing on first BotConnectedFrame.""" - if self._bot_connected_secs is not None or self._start_frame_arrival_ns is None: + if self._bot_connected_secs is not None or self._start_frame is None: return - delta_ns = data.timestamp - self._start_frame_arrival_ns + delta_ns = data.timestamp - self._start_frame.arrival_ns self._bot_connected_secs = delta_ns / 1e9 async def _handle_client_connected(self, data: FramePushed): """Emit transport timing report on first ClientConnectedFrame.""" - if self._transport_timing_reported or self._start_frame_arrival_ns is None: + if self._transport_timing_reported or self._start_frame is None: return self._transport_timing_reported = True - delta_ns = data.timestamp - self._start_frame_arrival_ns + delta_ns = data.timestamp - self._start_frame.arrival_ns client_connected_secs = delta_ns / 1e9 report = TransportTimingReport( - start_time=self._start_wall_clock or 0.0, + start_time=self._start_frame.wall_clock, bot_connected_secs=self._bot_connected_secs, client_connected_secs=client_connected_secs, ) @@ -319,7 +324,7 @@ class StartupTimingObserver(BaseObserver): total = sum(t.duration_secs for t in self._timings) report = StartupTimingReport( - start_time=self._start_wall_clock or 0.0, + start_time=self._start_frame.wall_clock if self._start_frame else 0.0, total_duration_secs=total, processor_timings=self._timings, ) From 31127abd9a2305c6119ef465e16a7a8351e12dde Mon Sep 17 00:00:00 2001 From: Daksh Dua Date: Thu, 16 Apr 2026 15:51:35 -0700 Subject: [PATCH 03/49] Allow inter-token whitespace once non-whitespace has been sent In token-streaming mode, _push_tts_frames previously stripped only leading newlines and dropped any pure-whitespace frame. That silently discarded meaningful inter-token whitespace (e.g. a standalone "\n" token between "hello" and "world"), losing prosody cues and any downstream sentence-boundary semantics. Track whether a non-whitespace character has been sent in the current context. While the flag is false, strip all leading whitespace; once true, let whitespace tokens flow through. Reset the flag on LLMFullResponseEndFrame/EndFrame and on interruption, and save/restore it around TTSSpeakFrame since each utterance is its own context. Sentence-aggregation mode preserves the existing behavior. --- src/pipecat/services/tts_service.py | 41 ++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/src/pipecat/services/tts_service.py b/src/pipecat/services/tts_service.py index fe4790cbb..65520129f 100644 --- a/src/pipecat/services/tts_service.py +++ b/src/pipecat/services/tts_service.py @@ -292,6 +292,7 @@ class TTSService(AIService): self._processing_text: bool = False self._tts_contexts: dict[str, TTSContext] = {} self._streamed_text: str = "" + self._sent_non_whitespace_in_context: bool = False self._text_aggregation_metrics_started: bool = False # Word timestamp state @@ -683,6 +684,7 @@ class TTSService(AIService): # Reset aggregator state self._processing_text = False + self._sent_non_whitespace_in_context = False if isinstance(frame, LLMFullResponseEndFrame): if self._push_text_frames: # Route through the serialization queue so the frame is @@ -697,6 +699,8 @@ class TTSService(AIService): elif isinstance(frame, TTSSpeakFrame): # Store if we were processing text or not so we can set it back. processing_text = self._processing_text + saved_sent_non_whitespace = self._sent_non_whitespace_in_context + self._sent_non_whitespace_in_context = False # TTSSpeakFrame is independent — temporarily clear the turn context # so create_context_id() generates a fresh UUID for this utterance. saved_turn_context_id = self._turn_context_id @@ -717,6 +721,7 @@ class TTSService(AIService): # the TTS. We pause to avoid audio overlapping. await self._maybe_pause_frame_processing() self._turn_context_id = saved_turn_context_id + self._sent_non_whitespace_in_context = saved_sent_non_whitespace self._processing_text = processing_text elif isinstance(frame, TTSUpdateSettingsFrame): if frame.service is not None and frame.service is not self: @@ -843,6 +848,7 @@ class TTSService(AIService): async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection): self._processing_text = False + self._sent_non_whitespace_in_context = False await self._text_aggregator.handle_interruption() for filter in self._text_filters: await filter.handle_interruption() @@ -900,13 +906,22 @@ class TTSService(AIService): await self.push_frame(src_frame) return - # Remove leading newlines only - text = text.lstrip("\n") - - # Don't send only whitespace. This causes problems for some TTS models. But also don't - # strip all whitespace, as whitespace can influence prosody. - if not text.strip(): - return + # Whitespace gating depends on aggregation mode: + # - Token streaming: drop all leading whitespace at the start of a context, as + # nothing substantive has been sent yet for it to attach to. Once a non-whitespace + # token has been sent, send whitespace as-is since it can influence prosody between + # non-whitespace tokens. + # + # - Sentence aggregation: strip leading newlines only and drop pure-whitespace frames. + if self._is_streaming_tokens: + if not self._sent_non_whitespace_in_context: + text = text.lstrip() + if not text: + return + else: + text = text.lstrip("\n") + if not text.strip(): + return # This is just a flag that indicates if we sent something to the TTS # service. It will be cleared if we sent text because of a TTSSpeakFrame @@ -928,9 +943,15 @@ class TTSService(AIService): await filter.reset_interruption() text = await filter.filter(text) - if not text.strip(): - if not self._is_streaming_tokens: - await self.stop_processing_metrics() + # Post-filter whitespace gate. Mirrors the pre-filter logic so filter + # output that collapses to whitespace-only is handled consistently. + if self._is_streaming_tokens: + # If empty, or only-whitespace and we haven't sent any non-whitespace, skip. + if not text or (not text.strip() and not self._sent_non_whitespace_in_context): + return + self._sent_non_whitespace_in_context = True + elif not text.strip(): + await self.stop_processing_metrics() return # Create context ID and store metadata From c6a18378441eacddd18393abd9b60940617bf1b1 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 16 Apr 2026 21:22:46 -0400 Subject: [PATCH 04/49] Fix type errors in extensions and add to pyright checked set Tighten LLMMessagesAppendFrame and LLMMessagesUpdateFrame message fields from list[dict] to list[LLMContextMessage] to match actual usage. Add type annotations on inline message lists in IVR navigator and voicemail detector. --- pyrightconfig.json | 3 ++- src/pipecat/extensions/ivr/ivr_navigator.py | 15 +++++++++------ .../extensions/voicemail/voicemail_detector.py | 6 +++--- src/pipecat/frames/frames.py | 9 ++++----- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/pyrightconfig.json b/pyrightconfig.json index 69d2ba8db..ea03dc37b 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -7,7 +7,8 @@ "src/pipecat/metrics", "src/pipecat/transcriptions", "src/pipecat/frames", - "src/pipecat/observers" + "src/pipecat/observers", + "src/pipecat/extensions" ], "exclude": [ "**/*_pb2.py", diff --git a/src/pipecat/extensions/ivr/ivr_navigator.py b/src/pipecat/extensions/ivr/ivr_navigator.py index f6b36655b..37b2469bb 100644 --- a/src/pipecat/extensions/ivr/ivr_navigator.py +++ b/src/pipecat/extensions/ivr/ivr_navigator.py @@ -30,6 +30,7 @@ from pipecat.frames.frames import ( VADParamsUpdateFrame, ) from pipecat.pipeline.pipeline import Pipeline +from pipecat.processors.aggregators.llm_context import LLMContextMessage from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.llm_service import LLMService from pipecat.utils.text.pattern_pair_aggregator import ( @@ -87,7 +88,7 @@ class IVRProcessor(FrameProcessor): self._classifier_prompt = classifier_prompt # Store saved context messages - self._saved_messages: list[dict] = [] + self._saved_messages: list[LLMContextMessage] = [] # XML pattern aggregation self._aggregator = PatternPairAggregator() @@ -97,18 +98,18 @@ class IVRProcessor(FrameProcessor): self._register_event_handler("on_conversation_detected") self._register_event_handler("on_ivr_status_changed") - def update_saved_messages(self, messages: list[dict]) -> None: + def update_saved_messages(self, messages: list[LLMContextMessage]) -> None: """Update the saved context messages. Sets the messages that are saved when switching between conversation and IVR navigation modes. Args: - messages: List of message dictionaries to save. + messages: List of context messages to save. """ self._saved_messages = messages - def _get_conversation_history(self) -> list[dict]: + def _get_conversation_history(self) -> list[LLMContextMessage]: """Get saved context messages without the system message. Returns: @@ -144,7 +145,9 @@ class IVRProcessor(FrameProcessor): await self.push_frame(frame, direction) # Set the classifier prompt and push it upstream - messages = [{"role": "system", "content": self._classifier_prompt}] + messages: list[LLMContextMessage] = [ + {"role": "developer", "content": self._classifier_prompt} + ] llm_update_frame = LLMMessagesUpdateFrame(messages=messages) await self.push_frame(llm_update_frame, FrameDirection.UPSTREAM) @@ -261,7 +264,7 @@ class IVRProcessor(FrameProcessor): logger.debug("IVR detected - switching to IVR navigation mode") # Create new context with IVR system prompt and saved messages - messages = [{"role": "system", "content": self._ivr_prompt}] + messages: list[LLMContextMessage] = [{"role": "developer", "content": self._ivr_prompt}] # Add saved conversation history if available conversation_history = self._get_conversation_history() diff --git a/src/pipecat/extensions/voicemail/voicemail_detector.py b/src/pipecat/extensions/voicemail/voicemail_detector.py index 3ab7f2f7d..d7e9d559b 100644 --- a/src/pipecat/extensions/voicemail/voicemail_detector.py +++ b/src/pipecat/extensions/voicemail/voicemail_detector.py @@ -35,7 +35,7 @@ from pipecat.frames.frames import ( UserStoppedSpeakingFrame, ) from pipecat.pipeline.parallel_pipeline import ParallelPipeline -from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_context import LLMContext, LLMContextMessage from pipecat.processors.aggregators.llm_response_universal import ( LLMContextAggregatorPair, LLMUserAggregatorParams, @@ -617,9 +617,9 @@ VOICEMAIL SYSTEM (respond "VOICEMAIL"): self._validate_prompt(custom_system_prompt) # Set up the LLM context with the classification prompt - self._messages = [ + self._messages: list[LLMContextMessage] = [ { - "role": "system", + "role": "developer", "content": self._prompt, }, ] diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 7fd215caf..24006428d 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -20,7 +20,6 @@ from typing import ( TYPE_CHECKING, Any, Literal, - Optional, ) from pipecat.adapters.schemas.tools_schema import ToolsSchema @@ -559,11 +558,11 @@ class LLMMessagesAppendFrame(DataFrame): current context. Parameters: - messages: List of message dictionaries to append. + messages: List of context messages to append. run_llm: Whether the context update should be sent to the LLM. """ - messages: list[dict] + messages: list[LLMContextMessage] run_llm: bool | None = None @@ -575,11 +574,11 @@ class LLMMessagesUpdateFrame(DataFrame): context LLM messages. Parameters: - messages: List of message dictionaries to replace current context. + messages: List of context messages to replace current context. run_llm: Whether the context update should be sent to the LLM. """ - messages: list[dict] + messages: list[LLMContextMessage] run_llm: bool | None = None From 36319ecbf082e84a89fac28fba70cc4305511890 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 16 Apr 2026 21:26:08 -0400 Subject: [PATCH 05/49] Replace system role message In UserTurnCompletionMixin, use a developer role message for LLM messages following an incomplete turn --- src/pipecat/turns/user_turn_completion_mixin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/turns/user_turn_completion_mixin.py b/src/pipecat/turns/user_turn_completion_mixin.py index 1d8da1e07..ab5397c06 100644 --- a/src/pipecat/turns/user_turn_completion_mixin.py +++ b/src/pipecat/turns/user_turn_completion_mixin.py @@ -292,7 +292,7 @@ class UserTurnCompletionLLMServiceMixin: # Push through pipeline to trigger LLM response await self.push_frame( - LLMMessagesAppendFrame(messages=[{"role": "system", "content": prompt}]) + LLMMessagesAppendFrame(messages=[{"role": "developer", "content": prompt}]) ) await self.push_frame(LLMRunFrame()) From 3127cc6161b4841c42edac1cd992d6d26cbcb6b8 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 16 Apr 2026 21:33:43 -0400 Subject: [PATCH 06/49] Fix type errors in turns and add to pyright checked set Widen base strategy process_frame return types to ProcessFrameResult | None to match actual behavior (None treated as CONTINUE). Give UserTurnCompletionLLMServiceMixin a FrameProcessor base class so pyright can see create_task, cancel_task, process_frame, and push_frame. --- pyrightconfig.json | 3 ++- .../turns/user_start/base_user_turn_start_strategy.py | 6 +++--- src/pipecat/turns/user_stop/base_user_turn_stop_strategy.py | 6 +++--- src/pipecat/turns/user_turn_completion_mixin.py | 4 ++-- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/pyrightconfig.json b/pyrightconfig.json index ea03dc37b..4016fd6e9 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -8,7 +8,8 @@ "src/pipecat/transcriptions", "src/pipecat/frames", "src/pipecat/observers", - "src/pipecat/extensions" + "src/pipecat/extensions", + "src/pipecat/turns" ], "exclude": [ "**/*_pb2.py", diff --git a/src/pipecat/turns/user_start/base_user_turn_start_strategy.py b/src/pipecat/turns/user_start/base_user_turn_start_strategy.py index 401178cc7..b424d7a9e 100644 --- a/src/pipecat/turns/user_start/base_user_turn_start_strategy.py +++ b/src/pipecat/turns/user_start/base_user_turn_start_strategy.py @@ -101,7 +101,7 @@ class BaseUserTurnStartStrategy(BaseObject): """Reset the strategy to its initial state.""" pass - async def process_frame(self, frame: Frame) -> ProcessFrameResult: + async def process_frame(self, frame: Frame) -> ProcessFrameResult | None: """Process an incoming frame. Subclasses should override this to implement logic that decides whether @@ -111,8 +111,8 @@ class BaseUserTurnStartStrategy(BaseObject): frame: The frame to be processed. Returns: - A ProcessFrameResult indicating the outcome. Subclasses that return - None are treated as CONTINUE for backward compatibility. + A ProcessFrameResult indicating the outcome, or None (treated as + CONTINUE for backward compatibility). """ pass diff --git a/src/pipecat/turns/user_stop/base_user_turn_stop_strategy.py b/src/pipecat/turns/user_stop/base_user_turn_stop_strategy.py index 1f8497359..e5dc1bd66 100644 --- a/src/pipecat/turns/user_stop/base_user_turn_stop_strategy.py +++ b/src/pipecat/turns/user_stop/base_user_turn_stop_strategy.py @@ -89,7 +89,7 @@ class BaseUserTurnStopStrategy(BaseObject): """Reset the strategy to its initial state.""" pass - async def process_frame(self, frame: Frame) -> ProcessFrameResult: + async def process_frame(self, frame: Frame) -> ProcessFrameResult | None: """Process an incoming frame to decide whether the user stopped speaking. Subclasses should override this to implement logic that decides whether @@ -99,8 +99,8 @@ class BaseUserTurnStopStrategy(BaseObject): frame: The frame to be analyzed. Returns: - A ProcessFrameResult indicating the outcome. Subclasses that return - None are treated as CONTINUE for backward compatibility. + A ProcessFrameResult indicating the outcome, or None (treated as + CONTINUE for backward compatibility). """ pass diff --git a/src/pipecat/turns/user_turn_completion_mixin.py b/src/pipecat/turns/user_turn_completion_mixin.py index ab5397c06..51d6c7828 100644 --- a/src/pipecat/turns/user_turn_completion_mixin.py +++ b/src/pipecat/turns/user_turn_completion_mixin.py @@ -26,7 +26,7 @@ from pipecat.frames.frames import ( LLMRunFrame, LLMTextFrame, ) -from pipecat.processors.frame_processor import FrameDirection +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor # Turn completion markers USER_TURN_COMPLETE_MARKER = "✓" @@ -178,7 +178,7 @@ class UserTurnCompletionConfig: return self.incomplete_long_prompt or DEFAULT_INCOMPLETE_LONG_PROMPT -class UserTurnCompletionLLMServiceMixin: +class UserTurnCompletionLLMServiceMixin(FrameProcessor): """Mixin that adds turn completion detection to LLM services. This mixin provides methods to push LLM text with turn completion detection. From ab91047300161fabbcca96d96bb8ffee65f3ac8e Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 16 Apr 2026 21:47:11 -0400 Subject: [PATCH 07/49] Fix type errors in pipeline and add to pyright checked set Use Sequence[FrameProcessor] instead of list[FrameProcessor] in Pipeline, ServiceSwitcher, and ServiceSwitcherStrategy parameters to accept subtype lists. Add cast() in LLMSwitcher for narrowed return types. Guard against None in task_observer._send_to_proxy and replace hasattr with truthiness check in task._cleanup. --- pyrightconfig.json | 3 ++- src/pipecat/pipeline/llm_switcher.py | 6 +++--- src/pipecat/pipeline/pipeline.py | 8 ++++---- src/pipecat/pipeline/service_switcher.py | 11 ++++++----- src/pipecat/pipeline/task.py | 2 +- src/pipecat/pipeline/task_observer.py | 2 ++ 6 files changed, 18 insertions(+), 14 deletions(-) diff --git a/pyrightconfig.json b/pyrightconfig.json index 4016fd6e9..7176ab774 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -9,7 +9,8 @@ "src/pipecat/frames", "src/pipecat/observers", "src/pipecat/extensions", - "src/pipecat/turns" + "src/pipecat/turns", + "src/pipecat/pipeline" ], "exclude": [ "**/*_pb2.py", diff --git a/src/pipecat/pipeline/llm_switcher.py b/src/pipecat/pipeline/llm_switcher.py index 71a7c7974..031f9e875 100644 --- a/src/pipecat/pipeline/llm_switcher.py +++ b/src/pipecat/pipeline/llm_switcher.py @@ -6,7 +6,7 @@ """LLM switcher for switching between different LLMs at runtime, with different switching strategies.""" -from typing import Any +from typing import Any, cast from pipecat.adapters.schemas.direct_function import DirectFunction from pipecat.pipeline.service_switcher import ( @@ -47,7 +47,7 @@ class LLMSwitcher(ServiceSwitcher[StrategyType]): Returns: List of LLM services managed by this switcher. """ - return self.services + return cast(list[LLMService], self.services) @property def active_llm(self) -> LLMService: @@ -56,7 +56,7 @@ class LLMSwitcher(ServiceSwitcher[StrategyType]): Returns: The currently active LLM service, or None if no LLM is active. """ - return self.strategy.active_service + return cast(LLMService, self.strategy.active_service) async def run_inference(self, context: LLMContext, **kwargs) -> str | None: """Run a one-shot, out-of-band (i.e. out-of-pipeline) inference with the given LLM context, using the currently active LLM. diff --git a/src/pipecat/pipeline/pipeline.py b/src/pipecat/pipeline/pipeline.py index 325cedb82..d7999b4fe 100644 --- a/src/pipecat/pipeline/pipeline.py +++ b/src/pipecat/pipeline/pipeline.py @@ -11,7 +11,7 @@ in sequence and manages frame flow between them, along with helper classes for pipeline source and sink operations. """ -from collections.abc import Callable, Coroutine +from collections.abc import Callable, Coroutine, Sequence from pipecat.frames.frames import Frame from pipecat.pipeline.base_pipeline import BasePipeline @@ -98,7 +98,7 @@ class Pipeline(BasePipeline): def __init__( self, - processors: list[FrameProcessor], + processors: Sequence[FrameProcessor], *, source: FrameProcessor | None = None, sink: FrameProcessor | None = None, @@ -106,7 +106,7 @@ class Pipeline(BasePipeline): """Initialize the pipeline with a list of processors. Args: - processors: List of frame processors to connect in sequence. + processors: Sequence of frame processors to connect in sequence. source: An optional pipeline source processor. sink: An optional pipeline sink processor. """ @@ -116,7 +116,7 @@ class Pipeline(BasePipeline): # downstream outside of the pipeline. self._source = source or PipelineSource(self.push_frame, name=f"{self}::Source") self._sink = sink or PipelineSink(self.push_frame, name=f"{self}::Sink") - self._processors: list[FrameProcessor] = [self._source] + processors + [self._sink] + self._processors: list[FrameProcessor] = [self._source, *processors, self._sink] self._link_processors() diff --git a/src/pipecat/pipeline/service_switcher.py b/src/pipecat/pipeline/service_switcher.py index a7f98f9b1..dd5e330e9 100644 --- a/src/pipecat/pipeline/service_switcher.py +++ b/src/pipecat/pipeline/service_switcher.py @@ -6,6 +6,7 @@ """Service switcher for switching between different services at runtime, with different switching strategies.""" +from collections.abc import Sequence from typing import Any, Generic, TypeVar from loguru import logger @@ -42,7 +43,7 @@ class ServiceSwitcherStrategy(BaseObject): ... """ - def __init__(self, services: list[FrameProcessor]): + def __init__(self, services: Sequence[FrameProcessor]): """Initialize the service switcher strategy with a list of services. Note: @@ -56,7 +57,7 @@ class ServiceSwitcherStrategy(BaseObject): if len(services) == 0: raise Exception(f"ServiceSwitcherStrategy needs at least one service") - self._services = services + self._services = list(services) self._active_service = services[0] self._register_event_handler("on_service_switched") @@ -223,7 +224,7 @@ class ServiceSwitcher(ParallelPipeline, Generic[StrategyType]): def __init__( self, - services: list[FrameProcessor], + services: Sequence[FrameProcessor], strategy_type: type[StrategyType] = ServiceSwitcherStrategyManual, ): """Initialize the service switcher with a list of services and a switching strategy. @@ -235,7 +236,7 @@ class ServiceSwitcher(ParallelPipeline, Generic[StrategyType]): """ _strategy = strategy_type(services) super().__init__(*self._make_pipeline_definitions(services, _strategy)) - self._services = services + self._services = list(services) self._strategy = _strategy @property @@ -250,7 +251,7 @@ class ServiceSwitcher(ParallelPipeline, Generic[StrategyType]): @staticmethod def _make_pipeline_definitions( - services: list[FrameProcessor], strategy: ServiceSwitcherStrategy + services: Sequence[FrameProcessor], strategy: ServiceSwitcherStrategy ) -> list[Any]: pipelines = [] for service in services: diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index 394b3d2e2..08e08ea00 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -742,7 +742,7 @@ class PipelineTask(BasePipelineTask): await self._observer.cleanup() # End conversation tracing if it's active - this will also close any active turn span - if self._enable_tracing and hasattr(self, "_turn_trace_observer"): + if self._enable_tracing and self._turn_trace_observer: self._turn_trace_observer.end_conversation_tracing() # Cleanup pipeline processors. diff --git a/src/pipecat/pipeline/task_observer.py b/src/pipecat/pipeline/task_observer.py index c6603c1d8..5b29850a6 100644 --- a/src/pipecat/pipeline/task_observer.py +++ b/src/pipecat/pipeline/task_observer.py @@ -173,6 +173,8 @@ class TaskObserver(BaseObserver): return proxies async def _send_to_proxy(self, data: Any): + if not self._proxies: + return for proxy in self._proxies.values(): await proxy.queue.put(data) From 80fecab4dea32de38966e83ff71bd4438cc1c297 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Fri, 17 Apr 2026 14:48:36 +0530 Subject: [PATCH 08/49] Fix SentryMetrics dropping MetricsFrame from stop_ttfb/stop_processing SentryMetrics.stop_ttfb_metrics and stop_processing_metrics called the base FrameProcessorMetrics implementation but discarded its return value (implicit `return None`). FrameProcessorMetrics.stop_ttfb_metrics / stop_processing_metrics build and return a MetricsFrame, which FrameProcessor.stop_ttfb_metrics / stop_processing_metrics then pushes downstream so observers (e.g. UserBotLatencyObserver, MetricsLogObserver) can see TTFB / processing metrics. Because SentryMetrics returned None, the FrameProcessor never pushed the MetricsFrame, so any pipeline using metrics=SentryMetrics() on STT / LLM / TTS services silently lost all downstream TTFB and processing MetricsFrames. The metrics were still calculated and logged internally, and Sentry transactions still finished correctly, but observers never saw them. Forward the MetricsFrame returned by the base class so FrameProcessor can push it into the pipeline. --- changelog/4325.fixed.md | 1 + src/pipecat/processors/metrics/sentry.py | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 changelog/4325.fixed.md diff --git a/changelog/4325.fixed.md b/changelog/4325.fixed.md new file mode 100644 index 000000000..b87a97bfb --- /dev/null +++ b/changelog/4325.fixed.md @@ -0,0 +1 @@ +- Fixed `SentryMetrics` silently dropping `MetricsFrame`s from `stop_ttfb_metrics` and `stop_processing_metrics`. `SentryMetrics` called the base `FrameProcessorMetrics` implementation but discarded its return value, so `FrameProcessor` never pushed the `MetricsFrame` downstream. This prevented observers (e.g. `UserBotLatencyObserver`, `MetricsLogObserver`) from seeing TTFB and processing metrics for any service using `metrics=SentryMetrics()`. The metrics were still calculated and Sentry transactions still completed — only the downstream frame push was affected. diff --git a/src/pipecat/processors/metrics/sentry.py b/src/pipecat/processors/metrics/sentry.py index b043b9058..0484995db 100644 --- a/src/pipecat/processors/metrics/sentry.py +++ b/src/pipecat/processors/metrics/sentry.py @@ -97,13 +97,20 @@ class SentryMetrics(FrameProcessorMetrics): Args: end_time: Optional end timestamp override. + + Returns: + MetricsFrame produced by the base class, or None if not measuring. + Returning the frame is required so ``FrameProcessor.stop_ttfb_metrics`` + can push it downstream to observers. """ - await super().stop_ttfb_metrics(end_time=end_time) + frame = await super().stop_ttfb_metrics(end_time=end_time) if self._sentry_available and self._ttfb_metrics_tx: await self._sentry_queue.put(self._ttfb_metrics_tx) self._ttfb_metrics_tx = None + return frame + async def start_processing_metrics(self, *, start_time: float | None = None): """Start tracking frame processing metrics. @@ -126,13 +133,20 @@ class SentryMetrics(FrameProcessorMetrics): Args: end_time: Optional end timestamp override. + + Returns: + MetricsFrame produced by the base class, or None if not measuring. + Returning the frame is required so ``FrameProcessor.stop_processing_metrics`` + can push it downstream to observers. """ - await super().stop_processing_metrics(end_time=end_time) + frame = await super().stop_processing_metrics(end_time=end_time) if self._sentry_available and self._processing_metrics_tx: await self._sentry_queue.put(self._processing_metrics_tx) self._processing_metrics_tx = None + return frame + async def _sentry_task_handler(self): """Background task handler for completing Sentry transactions.""" running = True From 4c19f5584cada7947d56f20de7a571b6cacbd7d5 Mon Sep 17 00:00:00 2001 From: Garegin Harutyunyan <14837795+realgarik@users.noreply.github.com> Date: Fri, 17 Apr 2026 15:53:41 +0400 Subject: [PATCH 09/49] VIVA SDK TT v3 support (#4252) * VIVA SDK TT v3 support * Format fix. * Renamed the API naming, removed '3' from the name. * Implementation of User turn start strategy using Krisp VIVA Interruption Prediction in scope of TT v3 support. * Typo fix in voice-krisp-viva example to use KrispVivaFilter class * style fix. * test run error fixes. * some test related changes. * Fixed tests * Stule fixes. --- examples/voice/voice-krisp-viva.py | 31 +- src/pipecat/audio/krisp_instance.py | 2 +- src/pipecat/audio/turn/krisp_viva_turn.py | 27 +- src/pipecat/turns/user_start/__init__.py | 6 + .../krisp_viva_ip_user_turn_start_strategy.py | 282 ++++++++++++++++++ .../test_krisp_ip_user_turn_start_strategy.py | 236 +++++++++++++++ tests/test_krisp_sdk_manager.py | 7 + 7 files changed, 564 insertions(+), 27 deletions(-) create mode 100644 src/pipecat/turns/user_start/krisp_viva_ip_user_turn_start_strategy.py create mode 100644 tests/test_krisp_ip_user_turn_start_strategy.py diff --git a/examples/voice/voice-krisp-viva.py b/examples/voice/voice-krisp-viva.py index 5fdefd2e1..06a236e0d 100644 --- a/examples/voice/voice-krisp-viva.py +++ b/examples/voice/voice-krisp-viva.py @@ -4,20 +4,24 @@ # SPDX-License-Identifier: BSD 2-Clause License # -"""Interruptible bot with Krisp VIVA noise filtering and turn detection. +"""Interruptible bot with Krisp VIVA noise filtering, turn detection, and IP. This example demonstrates a conversational bot with: - Krisp VIVA noise reduction on incoming audio -- Krisp VIVA Turn detection for natural interruptions +- Krisp VIVA Turn detection for end-of-turn +- Krisp Interruption Prediction (IP) to filter backchannels from real interruptions - Voice activity detection (VAD) Required environment variables: - KRISP_VIVA_FILTER_MODEL_PATH: Path to the Krisp noise filter model file (.kef) - KRISP_VIVA_TURN_MODEL_PATH: Path to the Krisp turn detection model file (.kef) -- DEEPGRAM_API_KEY: Deepgram API key for STT/TTS +- KRISP_VIVA_IP_MODEL_PATH: Path to the Krisp IP model file (.kef) +- DEEPGRAM_API_KEY: Deepgram API key for STT +- CARTESIA_API_KEY: Cartesia API key for TTS - OPENAI_API_KEY: OpenAI API key for LLM Optional environment variables: +- KRISP_VIVA_API_KEY: Krisp SDK API key (or set in code) - KRISP_NOISE_SUPPRESSION_LEVEL: Noise suppression level 0-100 (default: 100) Higher values = more aggressive noise reduction """ @@ -49,31 +53,30 @@ from pipecat.services.openai.llm import OpenAILLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams +from pipecat.turns.user_start import ( + KrispVivaIPUserTurnStartStrategy, + TranscriptionUserTurnStartStrategy, +) from pipecat.turns.user_stop import TurnAnalyzerUserTurnStopStrategy from pipecat.turns.user_turn_strategies import UserTurnStrategies load_dotenv(override=True) -# We use lambdas to defer transport parameter creation until the transport -# type is selected at runtime. - -krisp_viva_filter = KrispVivaFilter() - transport_params = { "daily": lambda: DailyParams( audio_in_enabled=True, audio_out_enabled=True, - audio_in_filter=krisp_viva_filter, + audio_in_filter=KrispVivaFilter(), ), "twilio": lambda: FastAPIWebsocketParams( audio_in_enabled=True, audio_out_enabled=True, - audio_in_filter=krisp_viva_filter, + audio_in_filter=KrispVivaFilter(), ), "webrtc": lambda: TransportParams( audio_in_enabled=True, audio_out_enabled=True, - audio_in_filter=krisp_viva_filter, + audio_in_filter=KrispVivaFilter(), ), } @@ -102,7 +105,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): context, user_params=LLMUserAggregatorParams( user_turn_strategies=UserTurnStrategies( - stop=[TurnAnalyzerUserTurnStopStrategy(turn_analyzer=KrispVivaTurn())] + start=[ + KrispVivaIPUserTurnStartStrategy(threshold=0.5), + TranscriptionUserTurnStartStrategy(), + ], + stop=[TurnAnalyzerUserTurnStopStrategy(turn_analyzer=KrispVivaTurn())], ), vad_analyzer=SileroVADAnalyzer(), # or KrispVivaVadAnalyzer ), diff --git a/src/pipecat/audio/krisp_instance.py b/src/pipecat/audio/krisp_instance.py index 5ebfd24cc..94e3dfe3f 100644 --- a/src/pipecat/audio/krisp_instance.py +++ b/src/pipecat/audio/krisp_instance.py @@ -17,7 +17,7 @@ try: except ModuleNotFoundError as e: logger.error(f"Exception: {e}") logger.error("In order to use the Krisp instance, you need to install krisp_audio.") - raise Exception(f"Missing module: {e}") + raise ImportError(f"Missing module: {e}") from e # Mapping of sample rates (Hz) to Krisp SDK SamplingRate enums diff --git a/src/pipecat/audio/turn/krisp_viva_turn.py b/src/pipecat/audio/turn/krisp_viva_turn.py index 5235a94be..a9cf6a847 100644 --- a/src/pipecat/audio/turn/krisp_viva_turn.py +++ b/src/pipecat/audio/turn/krisp_viva_turn.py @@ -7,7 +7,9 @@ """Krisp turn analyzer for end-of-turn detection using Krisp VIVA SDK. This module provides a turn analyzer implementation using Krisp's turn detection -(Tt) API to determine when a user has finished speaking in a conversation. +v3 (Tt) API to determine when a user has finished speaking in a conversation. +The Tt API accepts an external VAD flag alongside audio frames, allowing the +model to leverage voice activity information for more accurate turn detection. Note: This analyzer uses a different model than KrispVivaFilter. The model path can be specified via the KRISP_VIVA_TURN_MODEL_PATH environment variable or @@ -33,7 +35,7 @@ try: except ModuleNotFoundError as e: logger.error(f"Exception: {e}") logger.error("In order to use KrispVivaTurn, you need to install krisp_audio.") - raise Exception(f"Missing module: {e}") + raise ImportError(f"Missing module: {e}") from e class KrispTurnParams(BaseTurnParams): @@ -53,8 +55,10 @@ class KrispTurnParams(BaseTurnParams): class KrispVivaTurn(BaseTurnAnalyzer): """Turn analyzer using Krisp VIVA SDK for end-of-turn detection. - Uses Krisp's turn detection (Tt) API to determine when a user has finished - speaking. This analyzer requires a valid Krisp model file to operate. + Uses Krisp's turn detection v3 (Tt) API to determine when a user has + finished speaking. The Tt API receives an external VAD flag with each + audio frame, which the ``is_speech`` parameter of ``append_audio`` + provides. This analyzer requires a valid Krisp model file to operate. """ def __init__( @@ -158,14 +162,14 @@ class KrispVivaTurn(BaseTurnAnalyzer): """Create a turn detection session with the specified sample rate. Args: - sample_rate: Sample rate for the session + sample_rate: Sample rate for the session. Returns: - krisp_audio.TtFloat instance + krisp_audio.TtFloat instance. Raises: - ValueError: If sample rate or frame duration is not supported - RuntimeError: If session creation fails + ValueError: If sample rate or frame duration is not supported. + RuntimeError: If session creation fails. """ try: model_info = krisp_audio.ModelInfo() @@ -306,12 +310,7 @@ class KrispVivaTurn(BaseTurnAnalyzer): # Instead, we wait for the model's probability check below to confirm # end-of-turn based on the threshold. - prob = self._tt_session.process(frame.tolist()) - - # Negative values indicate the model is not ready yet (working with 100ms data) - # Skip processing until we get positive probabilities - if prob < 0: - continue + prob = self._tt_session.process(frame.tolist(), is_speech, False) # Store the probability for external access self._last_probability = prob diff --git a/src/pipecat/turns/user_start/__init__.py b/src/pipecat/turns/user_start/__init__.py index 94d12708d..14de5d28b 100644 --- a/src/pipecat/turns/user_start/__init__.py +++ b/src/pipecat/turns/user_start/__init__.py @@ -11,9 +11,15 @@ from .transcription_user_turn_start_strategy import TranscriptionUserTurnStartSt from .vad_user_turn_start_strategy import VADUserTurnStartStrategy from .wake_phrase_user_turn_start_strategy import WakePhraseUserTurnStartStrategy +try: + from .krisp_viva_ip_user_turn_start_strategy import KrispVivaIPUserTurnStartStrategy +except ImportError: + KrispVivaIPUserTurnStartStrategy = None + __all__ = [ "BaseUserTurnStartStrategy", "ExternalUserTurnStartStrategy", + "KrispVivaIPUserTurnStartStrategy", "MinWordsUserTurnStartStrategy", "TranscriptionUserTurnStartStrategy", "UserTurnStartedParams", diff --git a/src/pipecat/turns/user_start/krisp_viva_ip_user_turn_start_strategy.py b/src/pipecat/turns/user_start/krisp_viva_ip_user_turn_start_strategy.py new file mode 100644 index 000000000..807bc8154 --- /dev/null +++ b/src/pipecat/turns/user_start/krisp_viva_ip_user_turn_start_strategy.py @@ -0,0 +1,282 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""User turn start strategy using Krisp Interruption Prediction (IP). + +This strategy uses Krisp's IP model to distinguish genuine user interruptions +from backchannels (e.g. "uh-huh", "yeah"). Instead of triggering a user turn +on every VAD speech event, it collects audio after VAD detects speech and runs +the IP model to predict whether the speech is a real interruption. + +Only when the IP model's probability exceeds the configured threshold is +``trigger_user_turn_started()`` called. This prevents the bot from being +interrupted by brief acknowledgements or filler words. +""" + +import os + +import numpy as np +from loguru import logger + +from pipecat.audio.krisp_instance import ( + KrispVivaSDKManager, + int_to_krisp_frame_duration, + int_to_krisp_sample_rate, +) +from pipecat.frames.frames import ( + BotStoppedSpeakingFrame, + Frame, + InputAudioRawFrame, + VADUserStartedSpeakingFrame, + VADUserStoppedSpeakingFrame, +) +from pipecat.turns.types import ProcessFrameResult +from pipecat.turns.user_start.base_user_turn_start_strategy import BaseUserTurnStartStrategy + +try: + import krisp_audio +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error( + "In order to use KrispVivaIPUserTurnStartStrategy, you need to install krisp_audio." + ) + raise Exception(f"Missing module: {e}") + + +class KrispVivaIPUserTurnStartStrategy(BaseUserTurnStartStrategy): + """User turn start strategy using Krisp VIVA Interruption Prediction. + + When VAD detects user speech, this strategy feeds audio frames into + the Krisp VIVA IP model. The model outputs a probability indicating + whether the speech is a genuine interruption (as opposed to a + backchannel). A user turn is triggered only when this probability + exceeds the configured threshold. + + This strategy is designed to work alongside other start strategies + (e.g. ``TranscriptionUserTurnStartStrategy`` as a fallback) via the + strategy list in ``UserTurnStrategies``. + + Example:: + + from pipecat.turns.user_start import KrispVivaIPUserTurnStartStrategy + + strategies = UserTurnStrategies( + start=[ + KrispVivaIPUserTurnStartStrategy( + model_path="/path/to/ip_model.kef", + threshold=0.5, + ), + TranscriptionUserTurnStartStrategy(), + ], + ) + """ + + def __init__( + self, + *, + model_path: str | None = None, + threshold: float = 0.5, + frame_duration_ms: int = 20, + api_key: str = "", + **kwargs, + ): + """Initialize the Krisp VIVA IP user turn start strategy. + + Args: + model_path: Path to the Krisp VIVA IP model file (.kef). If None, + uses the KRISP_VIVA_IP_MODEL_PATH environment variable. + threshold: IP probability threshold (0.0 to 1.0). When the model's + output exceeds this value, the speech is classified as a genuine + interruption. + frame_duration_ms: Frame duration in milliseconds for IP processing. + Supported values: 10, 15, 20, 30, 32. + api_key: Krisp SDK API key. If empty, falls back to the + KRISP_VIVA_API_KEY environment variable. + **kwargs: Additional arguments passed to BaseUserTurnStartStrategy. + """ + super().__init__(**kwargs) + + self._threshold = threshold + self._frame_duration_ms = frame_duration_ms + self._api_key = api_key + + self._model_path = model_path or os.getenv("KRISP_VIVA_IP_MODEL_PATH") + if not self._model_path: + raise ValueError( + "IP model path must be provided via model_path or " + "KRISP_VIVA_IP_MODEL_PATH environment variable." + ) + if not self._model_path.endswith(".kef"): + raise ValueError("Model is expected with .kef extension") + if not os.path.isfile(self._model_path): + raise FileNotFoundError(f"IP model file not found: {self._model_path}") + + self._sdk_acquired = False + self._ip_session = None + self._samples_per_frame: int | None = None + self._sample_rate: int | None = None + + # State tracking + self._speech_active = False + self._audio_buffer = bytearray() + self._decision_made = False + + # Acquire SDK + try: + KrispVivaSDKManager.acquire(api_key=api_key) + self._sdk_acquired = True + except Exception as e: + raise RuntimeError(f"Failed to initialize Krisp SDK: {e}") + + async def cleanup(self): + """Release Krisp SDK resources.""" + if self._sdk_acquired: + try: + self._ip_session = None + KrispVivaSDKManager.release() + self._sdk_acquired = False + except Exception as e: + logger.error(f"Error cleaning up Krisp VIVA IP strategy: {e}", exc_info=True) + + def _ensure_session(self, sample_rate: int): + """Create or re-create the IP session when sample rate changes. + + Args: + sample_rate: Audio sample rate in Hz. + """ + if self._sample_rate == sample_rate and self._ip_session is not None: + return + + self._sample_rate = sample_rate + self._samples_per_frame = int((sample_rate * self._frame_duration_ms) / 1000) + + model_info = krisp_audio.ModelInfo() + model_info.path = self._model_path + + ip_cfg = krisp_audio.IpSessionConfig() + ip_cfg.inputSampleRate = int_to_krisp_sample_rate(sample_rate) + ip_cfg.inputFrameDuration = int_to_krisp_frame_duration(self._frame_duration_ms) + ip_cfg.modelInfo = model_info + + self._ip_session = krisp_audio.IpFloat.create(ip_cfg) + logger.debug(f"Krisp VIVA IP session created (sample_rate={sample_rate})") + + def _reset_state(self): + """Reset speech tracking state for the next candidate interruption.""" + self._speech_active = False + self._audio_buffer.clear() + self._decision_made = False + + async def reset(self): + """Reset the strategy to its initial state.""" + await super().reset() + self._reset_state() + + async def process_frame(self, frame: Frame) -> ProcessFrameResult: + """Process a frame to detect genuine user interruptions. + + On ``VADUserStartedSpeakingFrame``, begins collecting audio. + On ``InputAudioRawFrame``, feeds audio through the IP model and + triggers a user turn if the interruption probability exceeds the + threshold. + On ``VADUserStoppedSpeakingFrame`` or ``BotStoppedSpeakingFrame``, + resets the candidate state. + + Args: + frame: The incoming frame. + + Returns: + STOP if a genuine interruption was detected, CONTINUE otherwise. + """ + if isinstance(frame, VADUserStartedSpeakingFrame): + return await self._handle_vad_started(frame) + elif isinstance(frame, InputAudioRawFrame): + return await self._handle_audio(frame) + elif isinstance(frame, (VADUserStoppedSpeakingFrame, BotStoppedSpeakingFrame)): + return await self._handle_reset(frame) + + return ProcessFrameResult.CONTINUE + + async def _handle_vad_started(self, frame: VADUserStartedSpeakingFrame) -> ProcessFrameResult: + """Begin collecting audio for interruption classification. + + Args: + frame: The VAD speech-start frame. + + Returns: + Always CONTINUE; the decision is deferred until enough audio is processed. + """ + logger.trace("Krisp VIVA IP: VAD speech started, collecting audio for classification") + self._speech_active = True + self._audio_buffer.clear() + self._decision_made = False + return ProcessFrameResult.CONTINUE + + async def _handle_audio(self, frame: InputAudioRawFrame) -> ProcessFrameResult: + """Feed audio to the IP model and check for genuine interruption. + + Args: + frame: Raw audio input frame. + + Returns: + STOP if the model detects a genuine interruption, CONTINUE otherwise. + """ + if not self._speech_active or self._decision_made: + return ProcessFrameResult.CONTINUE + + self._ensure_session(frame.sample_rate) + + if self._ip_session is None or self._samples_per_frame is None: + logger.warning("IP session not ready, skipping frame") + return ProcessFrameResult.CONTINUE + + self._audio_buffer.extend(frame.audio) + + total_samples = len(self._audio_buffer) // 2 # 2 bytes per int16 sample + num_complete_frames = total_samples // self._samples_per_frame + + if num_complete_frames == 0: + return ProcessFrameResult.CONTINUE + + complete_samples_count = num_complete_frames * self._samples_per_frame + bytes_to_process = complete_samples_count * 2 + + audio_to_process = bytes(self._audio_buffer[:bytes_to_process]) + self._audio_buffer = self._audio_buffer[bytes_to_process:] + + audio_int16 = np.frombuffer(audio_to_process, dtype=np.int16) + audio_float32 = audio_int16.astype(np.float32) / 32768.0 + frames = audio_float32.reshape(-1, self._samples_per_frame) + + for ip_frame in frames: + ip_prob = self._ip_session.process(ip_frame.tolist(), self._speech_active) + + if ip_prob >= self._threshold: + logger.debug( + f"Krisp VIVA IP: genuine interruption detected (prob={ip_prob:.3f}, " + f"threshold={self._threshold})" + ) + self._decision_made = True + await self.trigger_user_turn_started() + return ProcessFrameResult.STOP + + return ProcessFrameResult.CONTINUE + + async def _handle_reset( + self, frame: VADUserStoppedSpeakingFrame | BotStoppedSpeakingFrame + ) -> ProcessFrameResult: + """Reset state when the candidate interruption window ends. + + Args: + frame: The frame signaling end of speech or bot output. + + Returns: + Always CONTINUE. + """ + if self._speech_active: + logger.trace("Krisp VIVA IP: speech segment ended, resetting state") + self._reset_state() + return ProcessFrameResult.CONTINUE diff --git a/tests/test_krisp_ip_user_turn_start_strategy.py b/tests/test_krisp_ip_user_turn_start_strategy.py new file mode 100644 index 000000000..bb34d879a --- /dev/null +++ b/tests/test_krisp_ip_user_turn_start_strategy.py @@ -0,0 +1,236 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os +import sys +import tempfile +import unittest +from unittest.mock import MagicMock, patch + +import numpy as np + +# Mock package version check before importing pipecat (development mode) +_version_patcher = patch("importlib.metadata.version", return_value="0.0.0-dev") +_version_patcher.start() + +# Mock krisp_audio before any pipecat import that loads krisp_instance / VIVA IP strategy +mock_krisp_audio = MagicMock() +mock_krisp_audio.SamplingRate.Sr8000Hz = 8000 +mock_krisp_audio.SamplingRate.Sr16000Hz = 16000 +mock_krisp_audio.SamplingRate.Sr24000Hz = 24000 +mock_krisp_audio.SamplingRate.Sr32000Hz = 32000 +mock_krisp_audio.SamplingRate.Sr44100Hz = 44100 +mock_krisp_audio.SamplingRate.Sr48000Hz = 48000 +mock_krisp_audio.FrameDuration.Fd10ms = "10ms" +mock_krisp_audio.FrameDuration.Fd15ms = "15ms" +mock_krisp_audio.FrameDuration.Fd20ms = "20ms" +mock_krisp_audio.FrameDuration.Fd30ms = "30ms" +mock_krisp_audio.FrameDuration.Fd32ms = "32ms" + +sys.modules["krisp_audio"] = mock_krisp_audio + +mock_pipecat_krisp = MagicMock() +sys.modules["pipecat_ai_krisp"] = mock_pipecat_krisp +sys.modules["pipecat_ai_krisp.audio"] = MagicMock() +sys.modules["pipecat_ai_krisp.audio.krisp_processor"] = MagicMock() + +from pipecat.frames.frames import ( + BotStartedSpeakingFrame, + BotStoppedSpeakingFrame, + InputAudioRawFrame, + TranscriptionFrame, + VADUserStartedSpeakingFrame, + VADUserStoppedSpeakingFrame, +) +from pipecat.turns.types import ProcessFrameResult +from pipecat.turns.user_start.krisp_viva_ip_user_turn_start_strategy import ( + KrispVivaIPUserTurnStartStrategy, +) + +STRATEGY_MODULE = "pipecat.turns.user_start.krisp_viva_ip_user_turn_start_strategy" + + +def _int16_silence(num_samples: int) -> bytes: + return np.zeros(num_samples, dtype=np.int16).tobytes() + + +class TestKrispVivaIPUserTurnStartStrategy(unittest.IsolatedAsyncioTestCase): + """Tests for KrispVivaIPUserTurnStartStrategy with mocked krisp_audio.""" + + def setUp(self): + self.temp_model_file = tempfile.NamedTemporaryFile(suffix=".kef", delete=False) + self.temp_model_file.write(b"dummy") + self.temp_model_file.close() + self.model_path = self.temp_model_file.name + + self.mock_krisp_audio = mock_krisp_audio + self.mock_krisp_audio.reset_mock() + self.mock_krisp_audio.ModelInfo.reset_mock() + self.mock_krisp_audio.IpSessionConfig.reset_mock() + self.mock_krisp_audio.IpFloat.reset_mock() + + self.mock_model_info = MagicMock() + self.mock_krisp_audio.ModelInfo.return_value = self.mock_model_info + + self.mock_ip_cfg = MagicMock() + self.mock_krisp_audio.IpSessionConfig.return_value = self.mock_ip_cfg + + self.mock_ip_session = MagicMock() + self.mock_krisp_audio.IpFloat.create.return_value = self.mock_ip_session + + self.krisp_patch = patch(f"{STRATEGY_MODULE}.krisp_audio", self.mock_krisp_audio) + self.krisp_patch.start() + + self.sdk_patcher = patch(f"{STRATEGY_MODULE}.KrispVivaSDKManager") + self.mock_sdk_manager = self.sdk_patcher.start() + self.mock_sdk_manager.acquire = MagicMock() + self.mock_sdk_manager.release = MagicMock() + + def tearDown(self): + self.krisp_patch.stop() + self.sdk_patcher.stop() + if os.path.exists(self.model_path): + os.unlink(self.model_path) + + def _make_strategy(self, *, threshold: float = 0.5, frame_duration_ms: int = 20): + return KrispVivaIPUserTurnStartStrategy( + model_path=self.model_path, + threshold=threshold, + frame_duration_ms=frame_duration_ms, + api_key="test-key", + ) + + def _audio_frame(self, sample_rate: int = 16000, frame_duration_ms: int = 20): + samples = int(sample_rate * frame_duration_ms / 1000) + return InputAudioRawFrame( + audio=_int16_silence(samples), + sample_rate=sample_rate, + num_channels=1, + ) + + async def test_interruption_detected_emits_turn_and_stop(self): + self.mock_ip_session.process = MagicMock(return_value=0.87) + + strategy = self._make_strategy(threshold=0.5) + try: + fired = False + + @strategy.event_handler("on_user_turn_started") + async def on_user_turn_started(strategy, params): + nonlocal fired + fired = True + + await strategy.process_frame(VADUserStartedSpeakingFrame()) + result = await strategy.process_frame(self._audio_frame()) + + self.assertTrue(fired) + self.assertEqual(result, ProcessFrameResult.STOP) + self.mock_ip_session.process.assert_called() + finally: + await strategy.cleanup() + + async def test_backchannel_suppressed_no_event_continue(self): + self.mock_ip_session.process = MagicMock(return_value=0.23) + + strategy = self._make_strategy(threshold=0.5) + try: + fired = False + + @strategy.event_handler("on_user_turn_started") + async def on_user_turn_started(strategy, params): + nonlocal fired + fired = True + + await strategy.process_frame(VADUserStartedSpeakingFrame()) + result = await strategy.process_frame(self._audio_frame()) + + self.assertFalse(fired) + self.assertEqual(result, ProcessFrameResult.CONTINUE) + finally: + await strategy.cleanup() + + async def test_reset_on_vad_stopped_clears_state(self): + self.mock_ip_session.process = MagicMock(return_value=0.1) + + strategy = self._make_strategy(threshold=0.5) + try: + await strategy.process_frame(VADUserStartedSpeakingFrame()) + await strategy.process_frame(self._audio_frame()) + self.mock_ip_session.process.reset_mock() + + await strategy.process_frame(VADUserStoppedSpeakingFrame()) + result = await strategy.process_frame(self._audio_frame()) + + self.assertEqual(result, ProcessFrameResult.CONTINUE) + self.mock_ip_session.process.assert_not_called() + finally: + await strategy.cleanup() + + async def test_reset_on_bot_stopped_clears_state(self): + self.mock_ip_session.process = MagicMock(return_value=0.1) + + strategy = self._make_strategy(threshold=0.5) + try: + await strategy.process_frame(VADUserStartedSpeakingFrame()) + await strategy.process_frame(self._audio_frame()) + self.mock_ip_session.process.reset_mock() + + await strategy.process_frame(BotStoppedSpeakingFrame()) + result = await strategy.process_frame(self._audio_frame()) + + self.assertEqual(result, ProcessFrameResult.CONTINUE) + self.mock_ip_session.process.assert_not_called() + finally: + await strategy.cleanup() + + async def test_no_op_before_vad_start(self): + self.mock_ip_session.process = MagicMock(return_value=0.99) + + strategy = self._make_strategy() + try: + result = await strategy.process_frame(self._audio_frame()) + self.assertEqual(result, ProcessFrameResult.CONTINUE) + self.mock_ip_session.process.assert_not_called() + finally: + await strategy.cleanup() + + async def test_decision_sticks_no_double_trigger(self): + self.mock_ip_session.process = MagicMock(return_value=0.9) + + strategy = self._make_strategy(threshold=0.5) + try: + count = 0 + + @strategy.event_handler("on_user_turn_started") + async def on_user_turn_started(strategy, params): + nonlocal count + count += 1 + + await strategy.process_frame(VADUserStartedSpeakingFrame()) + r1 = await strategy.process_frame(self._audio_frame()) + r2 = await strategy.process_frame(self._audio_frame()) + + self.assertEqual(r1, ProcessFrameResult.STOP) + self.assertEqual(r2, ProcessFrameResult.CONTINUE) + self.assertEqual(count, 1) + finally: + await strategy.cleanup() + + async def test_unrelated_frames_continue(self): + strategy = self._make_strategy() + try: + r1 = await strategy.process_frame(BotStartedSpeakingFrame()) + r2 = await strategy.process_frame( + TranscriptionFrame(text="hi", user_id="", timestamp="") + ) + self.assertEqual(r1, ProcessFrameResult.CONTINUE) + self.assertEqual(r2, ProcessFrameResult.CONTINUE) + finally: + await strategy.cleanup() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_krisp_sdk_manager.py b/tests/test_krisp_sdk_manager.py index 78a4d955f..2edbf4598 100644 --- a/tests/test_krisp_sdk_manager.py +++ b/tests/test_krisp_sdk_manager.py @@ -62,8 +62,15 @@ class TestKrispVivaSDKManager: def setup_method(self): """Reset mocks and SDK state before each test.""" mock_krisp_audio.reset_mock() + mock_krisp_audio.globalInit.side_effect = None mock_krisp_audio.getVersion.return_value = mock_version + # Ensure krisp_instance module uses THIS test's mock, not a stale + # reference cached from a different test file's sys.modules entry. + import pipecat.audio.krisp_instance as _ki + + _ki.krisp_audio = mock_krisp_audio + # Reset the SDK manager state for clean tests # We access internal state to ensure tests are isolated with KrispVivaSDKManager._lock: From cb1463f9f125ae01fbdaa619fd38ca8765b65770 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 16 Apr 2026 22:05:01 -0400 Subject: [PATCH 10/49] Fix type errors in runner and add to pyright checked set Make required parameters non-optional: LiveKitRunnerArguments.token, _create_telephony_transport args. Use os.environ[] instead of os.getenv() for required WhatsApp env vars. Guard spec/loader None in module loading. Tighten sip_caller_phone guard in daily.py. --- pyrightconfig.json | 3 ++- src/pipecat/runner/daily.py | 2 +- src/pipecat/runner/run.py | 35 ++++++++++++++++------------------- src/pipecat/runner/types.py | 2 +- src/pipecat/runner/utils.py | 12 +++--------- 5 files changed, 23 insertions(+), 31 deletions(-) diff --git a/pyrightconfig.json b/pyrightconfig.json index 7176ab774..17359ed4a 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -10,7 +10,8 @@ "src/pipecat/observers", "src/pipecat/extensions", "src/pipecat/turns", - "src/pipecat/pipeline" + "src/pipecat/pipeline", + "src/pipecat/runner" ], "exclude": [ "**/*_pb2.py", diff --git a/src/pipecat/runner/daily.py b/src/pipecat/runner/daily.py index bc80f0641..168b2353d 100644 --- a/src/pipecat/runner/daily.py +++ b/src/pipecat/runner/daily.py @@ -228,7 +228,7 @@ async def configure( room_properties.enable_dialout = True # Add SIP configuration if enabled - if sip_enabled: + if sip_enabled and sip_caller_phone: sip_params = DailyRoomSipParams( display_name=sip_caller_phone, video=sip_enable_video, diff --git a/src/pipecat/runner/run.py b/src/pipecat/runner/run.py index c6d43fbbd..ca296bd78 100644 --- a/src/pipecat/runner/run.py +++ b/src/pipecat/runner/run.py @@ -156,6 +156,8 @@ def _get_bot_module(): spec = importlib.util.spec_from_file_location( module_name, os.path.join(cwd, filename) ) + if spec is None or spec.loader is None: + continue module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) @@ -386,29 +388,24 @@ def _add_lifespan_to_app(app: FastAPI, new_lifespan): def _setup_whatsapp_routes(app: FastAPI, args: argparse.Namespace): """Set up WhatsApp-specific routes.""" - WHATSAPP_APP_SECRET = os.getenv("WHATSAPP_APP_SECRET") - WHATSAPP_PHONE_NUMBER_ID = os.getenv("WHATSAPP_PHONE_NUMBER_ID") - WHATSAPP_TOKEN = os.getenv("WHATSAPP_TOKEN") - WHATSAPP_WEBHOOK_VERIFICATION_TOKEN = os.getenv("WHATSAPP_WEBHOOK_VERIFICATION_TOKEN") - - if not all( - [ - WHATSAPP_APP_SECRET, - WHATSAPP_PHONE_NUMBER_ID, - WHATSAPP_TOKEN, - WHATSAPP_WEBHOOK_VERIFICATION_TOKEN, - ] - ): + required_vars = [ + "WHATSAPP_APP_SECRET", + "WHATSAPP_PHONE_NUMBER_ID", + "WHATSAPP_TOKEN", + "WHATSAPP_WEBHOOK_VERIFICATION_TOKEN", + ] + missing = [v for v in required_vars if not os.getenv(v)] + if missing: logger.error( - """Missing required environment variables for WhatsApp transport: - WHATSAPP_APP_SECRET - WHATSAPP_PHONE_NUMBER_ID - WHATSAPP_TOKEN - WHATSAPP_WEBHOOK_VERIFICATION_TOKEN - """ + f"Missing required environment variables for WhatsApp transport: {', '.join(missing)}" ) return + WHATSAPP_APP_SECRET = os.environ["WHATSAPP_APP_SECRET"] + WHATSAPP_PHONE_NUMBER_ID = os.environ["WHATSAPP_PHONE_NUMBER_ID"] + WHATSAPP_TOKEN = os.environ["WHATSAPP_TOKEN"] + WHATSAPP_WEBHOOK_VERIFICATION_TOKEN = os.environ["WHATSAPP_WEBHOOK_VERIFICATION_TOKEN"] + try: from pipecat.transports.smallwebrtc.connection import SmallWebRTCConnection from pipecat.transports.whatsapp.api import WhatsAppWebhookRequest diff --git a/src/pipecat/runner/types.py b/src/pipecat/runner/types.py index 055824a22..b009b3350 100644 --- a/src/pipecat/runner/types.py +++ b/src/pipecat/runner/types.py @@ -122,4 +122,4 @@ class LiveKitRunnerArguments(RunnerArguments): room_name: str url: str - token: str | None = None + token: str diff --git a/src/pipecat/runner/utils.py b/src/pipecat/runner/utils.py index 7a4b3034c..84316a743 100644 --- a/src/pipecat/runner/utils.py +++ b/src/pipecat/runner/utils.py @@ -416,9 +416,9 @@ def _get_transport_params(transport_key: str, transport_params: dict[str, Callab async def _create_telephony_transport( websocket: WebSocket, - params: Any | None = None, - transport_type: str = None, - call_data: dict = None, + params: Any, + transport_type: str, + call_data: dict, ) -> BaseTransport: """Create a telephony transport with pre-parsed WebSocket data. @@ -433,12 +433,6 @@ async def _create_telephony_transport( """ from pipecat.transports.websocket.fastapi import FastAPIWebsocketTransport - if params is None: - raise ValueError( - "FastAPIWebsocketParams must be provided. " - "The serializer and add_wav_header will be set automatically." - ) - # Always set add_wav_header to False for telephony params.add_wav_header = False From f5f92dea6309ca20591e9dfb115ec50602d5fc38 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 16 Apr 2026 22:55:50 -0400 Subject: [PATCH 11/49] Add changelog entries and restore multi-line WhatsApp error log Add changelog entries for the pyright introduction and the LiveKitRunnerArguments.token signature tightening. Restore the indented multi-line format for the WhatsApp missing-env error, now listing only the vars that are actually missing. --- changelog/4324.added.md | 1 + changelog/4324.changed.md | 1 + src/pipecat/runner/run.py | 5 ++++- 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 changelog/4324.added.md create mode 100644 changelog/4324.changed.md diff --git a/changelog/4324.added.md b/changelog/4324.added.md new file mode 100644 index 000000000..d38887585 --- /dev/null +++ b/changelog/4324.added.md @@ -0,0 +1 @@ +- Added incremental `pyright` type checking. A `pyrightconfig.json` at the repo root uses `typeCheckingMode: "basic"` with an explicit `include` list of modules that pass cleanly (`clocks`, `metrics`, `transcriptions`, `frames`, `observers`, `extensions`, `turns`, `pipeline`, `runner`). Remaining modules will be added in subsequent PRs. CI enforces the checked set via `uv run pyright` in the format workflow. diff --git a/changelog/4324.changed.md b/changelog/4324.changed.md new file mode 100644 index 000000000..672c60b62 --- /dev/null +++ b/changelog/4324.changed.md @@ -0,0 +1 @@ +- `LiveKitRunnerArguments.token` is now a required `str` (previously `str | None` with a default of `None`). LiveKit requires a token to join a room, so the type now reflects reality. This only affects custom runners that construct `LiveKitRunnerArguments` directly; code consuming the argument from the standard runner is unaffected. diff --git a/src/pipecat/runner/run.py b/src/pipecat/runner/run.py index ca296bd78..a53dea519 100644 --- a/src/pipecat/runner/run.py +++ b/src/pipecat/runner/run.py @@ -396,8 +396,11 @@ def _setup_whatsapp_routes(app: FastAPI, args: argparse.Namespace): ] missing = [v for v in required_vars if not os.getenv(v)] if missing: + missing_list = "\n ".join(missing) logger.error( - f"Missing required environment variables for WhatsApp transport: {', '.join(missing)}" + f"""Missing required environment variables for WhatsApp transport: + {missing_list} + """ ) return From 6bb4e8295f71e7551b93e29d8fd980df9c0d918e Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 17 Apr 2026 10:30:45 -0400 Subject: [PATCH 12/49] Add multilingual support for Deepgram Flux STT Enables the flux-general-multi model with one or more language_hints. Hints are sent as repeatable URL params at connect time and via a Configure control message when updated mid-stream (detect-then-lock). TranscriptionFrame.language now reflects the language Flux detected for each turn via the TurnInfo `languages` field. --- .../update-settings/stt/stt-deepgram-flux.py | 18 ++- src/pipecat/services/deepgram/flux/base.py | 113 ++++++++++++++++-- .../services/deepgram/flux/sagemaker/stt.py | 4 +- src/pipecat/services/deepgram/flux/stt.py | 20 +++- 4 files changed, 141 insertions(+), 14 deletions(-) diff --git a/examples/update-settings/stt/stt-deepgram-flux.py b/examples/update-settings/stt/stt-deepgram-flux.py index 24bb8b7ad..82e60ce67 100644 --- a/examples/update-settings/stt/stt-deepgram-flux.py +++ b/examples/update-settings/stt/stt-deepgram-flux.py @@ -23,6 +23,7 @@ from pipecat.processors.aggregators.llm_response_universal import ( from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.flux.base import DeepgramFluxSTTSettings from pipecat.services.deepgram.flux.stt import DeepgramFluxSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.transcriptions.language import Language @@ -51,7 +52,16 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramFluxSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + # Start with the multilingual model and broad hints so Flux can auto-detect + # across English and Spanish. TranscriptionFrame.language will reflect + # whichever language Flux detected for each turn. + stt = DeepgramFluxSTTService( + api_key=os.getenv("DEEPGRAM_API_KEY"), + settings=DeepgramFluxSTTService.Settings( + model="flux-general-multi", + language_hints=[Language.EN, Language.ES], + ), + ) tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), @@ -115,10 +125,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) ) + # Detect-then-lock: narrow the hints to a single language mid-stream. + # Sent as a Configure message — no reconnect needed. await asyncio.sleep(10) - logger.info("Updating Deepgram Flux STT settings: language=es") + logger.info("Updating Deepgram Flux STT settings: language_hints=[es]") await task.queue_frame( - STTUpdateSettingsFrame(delta=DeepgramFluxSTTService.Settings(language=Language.ES)) + STTUpdateSettingsFrame(delta=DeepgramFluxSTTSettings(language_hints=[Language.ES])) ) @transport.event_handler("on_client_disconnected") diff --git a/src/pipecat/services/deepgram/flux/base.py b/src/pipecat/services/deepgram/flux/base.py index baefbd060..089b5525a 100644 --- a/src/pipecat/services/deepgram/flux/base.py +++ b/src/pipecat/services/deepgram/flux/base.py @@ -27,11 +27,59 @@ from pipecat.frames.frames import ( ) from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven from pipecat.services.stt_service import STTService -from pipecat.transcriptions.language import Language +from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.time import time_now_iso8601 from pipecat.utils.tracing.service_decorators import traced_stt +def language_to_deepgram_flux_language(language: Language) -> str | None: + """Convert a Pipecat Language to a Deepgram Flux language code. + + Only honored by the ``flux-general-multi`` model. Locale variants + (e.g. ``Language.EN_GB``) fall back to the base code. + """ + LANGUAGE_MAP = { + Language.DE: "de", + Language.EN: "en", + Language.ES: "es", + Language.FR: "fr", + Language.HI: "hi", + Language.IT: "it", + Language.JA: "ja", + Language.NL: "nl", + Language.PT: "pt", + Language.RU: "ru", + } + return resolve_language(language, LANGUAGE_MAP, use_base_code=True) + + +def _prepare_language_hints(hints: list[Language] | None) -> list[str]: + """Convert a list of Pipecat Languages to Deepgram Flux codes. + + Drops entries that can't be mapped and deduplicates while preserving order. + """ + if not hints: + return [] + seen: set[str] = set() + prepared: list[str] = [] + for hint in hints: + code = language_to_deepgram_flux_language(hint) + if code is None or code in seen: + continue + seen.add(code) + prepared.append(code) + return prepared + + +def _code_to_pipecat_language(code: str) -> Language | None: + """Convert a Deepgram-returned language code to a Pipecat Language.""" + try: + return Language(code) + except ValueError: + logger.debug(f"Unmapped Deepgram Flux detected language code: {code}") + return None + + class FluxMessageType(StrEnum): """Deepgram Flux WebSocket message types. @@ -73,6 +121,10 @@ class DeepgramFluxSTTSettings(STTSettings): confidence (default 5000). keyterm: Keyterms to boost recognition accuracy for specialized terminology. min_confidence: Minimum confidence required to create a TranscriptionFrame. + language_hints: Languages to bias transcription toward. Only honored by the + ``flux-general-multi`` model. An empty list clears any active hints; + ``None``/``NOT_GIVEN`` means no hints (auto-detect). Can be updated + mid-stream via ``STTUpdateSettingsFrame``. """ eager_eot_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) @@ -80,6 +132,7 @@ class DeepgramFluxSTTSettings(STTSettings): eot_timeout_ms: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) keyterm: list | _NotGiven = field(default_factory=lambda: NOT_GIVEN) min_confidence: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + language_hints: list[Language] | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) class DeepgramFluxSTTBase(STTService): @@ -93,7 +146,14 @@ class DeepgramFluxSTTBase(STTService): Settings = DeepgramFluxSTTSettings _settings: Settings - _CONFIGURE_FIELDS = {"keyterm", "eot_threshold", "eager_eot_threshold", "eot_timeout_ms"} + _CONFIGURE_FIELDS = { + "keyterm", + "eot_threshold", + "eager_eot_threshold", + "eot_timeout_ms", + "language_hints", + } + _MULTILINGUAL_MODEL = "flux-general-multi" def __init__( self, @@ -200,6 +260,18 @@ class DeepgramFluxSTTBase(STTService): for tag_value in self._tag: params.append(urlencode({"tag": tag_value})) + # Add language_hint parameters (only valid on flux-general-multi) + hints = self._settings.language_hints + if hints and not isinstance(hints, _NotGiven): + if self._settings.model == self._MULTILINGUAL_MODEL: + for code in _prepare_language_hints(hints): + params.append(urlencode({"language_hint": code})) + else: + logger.warning( + f"language_hints only supported on {self._MULTILINGUAL_MODEL}; " + f"ignoring hints for model {self._settings.model!r}" + ) + return "&".join(params) async def _send_silence(self, duration_secs: float = 0.5): @@ -266,6 +338,21 @@ class DeepgramFluxSTTBase(STTService): if thresholds: message["thresholds"] = thresholds + if "language_hints" in fields: + if self._settings.model != self._MULTILINGUAL_MODEL: + logger.warning( + f"language_hints only supported on {self._MULTILINGUAL_MODEL}; " + f"skipping Configure update for model {self._settings.model!r}" + ) + else: + hints = self._settings.language_hints + # Empty list clears hints; NOT_GIVEN/None also treated as clear + # since we only reach this branch when the user set the field. + if hints is None or isinstance(hints, _NotGiven): + message["language_hints"] = [] + else: + message["language_hints"] = _prepare_language_hints(hints) + logger.debug(f"{self}: sending Configure message: {message}") await self._transport_send_json(message) @@ -281,8 +368,9 @@ class DeepgramFluxSTTBase(STTService): """Apply a settings delta. Configure-able fields (keyterm, eot_threshold, eager_eot_threshold, - eot_timeout_ms) are sent to Deepgram via a Configure message. - Other fields are stored but cannot be applied to the active connection. + eot_timeout_ms, language_hints) are sent to Deepgram via a Configure + message. Other fields are stored but cannot be applied to the active + connection. """ changed = await super()._update_settings(delta) @@ -520,6 +608,16 @@ class DeepgramFluxSTTBase(STTService): return None return sum(confidences) / len(confidences) + def _primary_detected_language(self, data: dict[str, Any]) -> Language | None: + """Extract the primary detected language from a TurnInfo payload. + + Only populated by ``flux-general-multi``; returns ``None`` otherwise. + """ + codes = data.get("languages") or [] + if not codes: + return None + return _code_to_pipecat_language(codes[0]) + async def _handle_end_of_turn(self, transcript: str, data: dict[str, Any]): """Handle EndOfTurn events from Deepgram Flux. @@ -543,6 +641,7 @@ class DeepgramFluxSTTBase(STTService): # Compute the average confidence average_confidence = self._calculate_average_confidence(data) + detected_language = self._primary_detected_language(data) if not self._settings.min_confidence or average_confidence > self._settings.min_confidence: # EndOfTurn means Flux has determined the turn is complete, @@ -552,7 +651,7 @@ class DeepgramFluxSTTBase(STTService): transcript, self._user_id, time_now_iso8601(), - self._settings.language, + detected_language, result=data, finalized=True, ) @@ -562,7 +661,7 @@ class DeepgramFluxSTTBase(STTService): f"Transcription confidence below min_confidence threshold: {average_confidence}" ) - await self._handle_transcription(transcript, True, self._settings.language) + await self._handle_transcription(transcript, True, detected_language) await self.stop_processing_metrics() await self.broadcast_frame(UserStoppedSpeakingFrame) await self._call_event_handler("on_end_of_turn", transcript) @@ -606,7 +705,7 @@ class DeepgramFluxSTTBase(STTService): transcript, self._user_id, time_now_iso8601(), - self._settings.language, + self._primary_detected_language(data), result=data, ) ) diff --git a/src/pipecat/services/deepgram/flux/sagemaker/stt.py b/src/pipecat/services/deepgram/flux/sagemaker/stt.py index da61b169a..94738fc9c 100644 --- a/src/pipecat/services/deepgram/flux/sagemaker/stt.py +++ b/src/pipecat/services/deepgram/flux/sagemaker/stt.py @@ -23,7 +23,6 @@ from pipecat.services.deepgram.flux.base import ( DeepgramFluxSTTBase, DeepgramFluxSTTSettings, ) -from pipecat.transcriptions.language import Language @dataclass @@ -112,12 +111,13 @@ class DeepgramFluxSageMakerSTTService(DeepgramFluxSTTBase): # Initialize default settings default_settings = self.Settings( model="flux-general-en", - language=Language.EN, + language=None, eager_eot_threshold=None, eot_threshold=None, eot_timeout_ms=None, keyterm=[], min_confidence=None, + language_hints=None, ) # Apply settings delta diff --git a/src/pipecat/services/deepgram/flux/stt.py b/src/pipecat/services/deepgram/flux/stt.py index 5b0b16472..6874b1b69 100644 --- a/src/pipecat/services/deepgram/flux/stt.py +++ b/src/pipecat/services/deepgram/flux/stt.py @@ -24,7 +24,6 @@ from pipecat.services.deepgram.flux.base import ( FluxMessageType, ) from pipecat.services.websocket_service import WebsocketService -from pipecat.transcriptions.language import Language try: from websockets.asyncio.client import connect as websocket_connect @@ -50,6 +49,12 @@ class DeepgramFluxSTTService(DeepgramFluxSTTBase, WebsocketService): Supports configurable models, VAD events, and various audio processing options including advanced turn detection and EagerEndOfTurn events for improved conversational AI performance. + For multilingual use, set ``model="flux-general-multi"`` and pass + ``language_hints`` to bias detection toward specific languages. Hints can + be updated mid-stream via ``STTUpdateSettingsFrame`` (e.g. to implement a + detect-then-lock flow). ``TranscriptionFrame.language`` reflects whichever + language Flux detected for each turn. + Event handlers available (in addition to base events): - on_start_of_turn(service, transcript): Deepgram detected start of speech @@ -156,6 +161,16 @@ class DeepgramFluxSTTService(DeepgramFluxSTTBase, WebsocketService): tag=["production", "voice-agent"], ), ) + + Multilingual usage with language hints:: + + stt = DeepgramFluxSTTService( + api_key="your-api-key", + settings=DeepgramFluxSTTService.Settings( + model="flux-general-multi", + language_hints=[Language.EN, Language.ES], + ), + ) """ # Note: For DeepgramFluxSTTService, differently from other processes, we need to create # the _receive_task inside _connect_websocket, because the websocket should only be @@ -171,12 +186,13 @@ class DeepgramFluxSTTService(DeepgramFluxSTTBase, WebsocketService): # 1. Initialize default_settings with hardcoded defaults default_settings = self.Settings( model="flux-general-en", - language=Language.EN, + language=None, eager_eot_threshold=None, eot_threshold=None, eot_timeout_ms=None, keyterm=[], min_confidence=None, + language_hints=None, ) # 2. Apply direct init arg overrides (deprecated) From af861b7975c2f2e443ba776704cc2dcec6303892 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 17 Apr 2026 10:31:37 -0400 Subject: [PATCH 13/49] Add changelog for #4326 --- changelog/4326.added.md | 1 + changelog/4326.changed.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog/4326.added.md create mode 100644 changelog/4326.changed.md diff --git a/changelog/4326.added.md b/changelog/4326.added.md new file mode 100644 index 000000000..856c7b5d7 --- /dev/null +++ b/changelog/4326.added.md @@ -0,0 +1 @@ +- Added multilingual support to `DeepgramFluxSTTService` via a new `language_hints: list[Language]` setting. Works with Deepgram's new `flux-general-multi` model to bias transcription across English, Spanish, French, German, Hindi, Russian, Portuguese, Japanese, Italian, and Dutch. Omit the hints to use auto-detection, or pass a subset to bias toward expected languages. Hints can be updated mid-stream via `STTUpdateSettingsFrame` (sent as a Deepgram `Configure` control message, no reconnect) to support detect-then-lock flows. diff --git a/changelog/4326.changed.md b/changelog/4326.changed.md new file mode 100644 index 000000000..2809777fc --- /dev/null +++ b/changelog/4326.changed.md @@ -0,0 +1 @@ +- `TranscriptionFrame.language` and `InterimTranscriptionFrame.language` emitted by `DeepgramFluxSTTService` now reflect the language Deepgram detected for each turn (read from the `languages` field on Flux's `TurnInfo` event) instead of a statically configured value. On `flux-general-multi` this means per-turn accuracy for downstream consumers (e.g. TTS voice selection). On `flux-general-en` the field is absent in Flux responses, so emitted frames now carry `language=None` instead of the previously hardcoded `Language.EN`. From 0340e25e9feef6d295162320b8fb9a97885b9008 Mon Sep 17 00:00:00 2001 From: filipi87 Date: Fri, 17 Apr 2026 12:44:57 -0300 Subject: [PATCH 14/49] Fixing typecheck for service switcher. --- scripts/{fix-ruff.sh => fix-ruff-and-typecheck.sh} | 4 ++++ src/pipecat/pipeline/llm_switcher.py | 3 ++- src/pipecat/pipeline/service_switcher.py | 11 +++++------ 3 files changed, 11 insertions(+), 7 deletions(-) rename scripts/{fix-ruff.sh => fix-ruff-and-typecheck.sh} (83%) diff --git a/scripts/fix-ruff.sh b/scripts/fix-ruff-and-typecheck.sh similarity index 83% rename from scripts/fix-ruff.sh rename to scripts/fix-ruff-and-typecheck.sh index 96c6a278b..31cce6dc3 100755 --- a/scripts/fix-ruff.sh +++ b/scripts/fix-ruff-and-typecheck.sh @@ -6,5 +6,9 @@ PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" echo "Running ruff format..." uv run ruff format "$PROJECT_ROOT" + echo "Running ruff check..." uv run ruff check --fix "$PROJECT_ROOT" + +echo "Running pyright check..." +uv run pyright diff --git a/src/pipecat/pipeline/llm_switcher.py b/src/pipecat/pipeline/llm_switcher.py index 031f9e875..9c89641e3 100644 --- a/src/pipecat/pipeline/llm_switcher.py +++ b/src/pipecat/pipeline/llm_switcher.py @@ -15,6 +15,7 @@ from pipecat.pipeline.service_switcher import ( StrategyType, ) from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.frame_processor import FrameProcessor from pipecat.services.llm_service import LLMService @@ -38,7 +39,7 @@ class LLMSwitcher(ServiceSwitcher[StrategyType]): strategy_type: The strategy class to use for switching between LLMs. Defaults to ``ServiceSwitcherStrategyManual``. """ - super().__init__(llms, strategy_type) + super().__init__(cast(list[FrameProcessor], llms), strategy_type) @property def llms(self) -> list[LLMService]: diff --git a/src/pipecat/pipeline/service_switcher.py b/src/pipecat/pipeline/service_switcher.py index dd5e330e9..a7f98f9b1 100644 --- a/src/pipecat/pipeline/service_switcher.py +++ b/src/pipecat/pipeline/service_switcher.py @@ -6,7 +6,6 @@ """Service switcher for switching between different services at runtime, with different switching strategies.""" -from collections.abc import Sequence from typing import Any, Generic, TypeVar from loguru import logger @@ -43,7 +42,7 @@ class ServiceSwitcherStrategy(BaseObject): ... """ - def __init__(self, services: Sequence[FrameProcessor]): + def __init__(self, services: list[FrameProcessor]): """Initialize the service switcher strategy with a list of services. Note: @@ -57,7 +56,7 @@ class ServiceSwitcherStrategy(BaseObject): if len(services) == 0: raise Exception(f"ServiceSwitcherStrategy needs at least one service") - self._services = list(services) + self._services = services self._active_service = services[0] self._register_event_handler("on_service_switched") @@ -224,7 +223,7 @@ class ServiceSwitcher(ParallelPipeline, Generic[StrategyType]): def __init__( self, - services: Sequence[FrameProcessor], + services: list[FrameProcessor], strategy_type: type[StrategyType] = ServiceSwitcherStrategyManual, ): """Initialize the service switcher with a list of services and a switching strategy. @@ -236,7 +235,7 @@ class ServiceSwitcher(ParallelPipeline, Generic[StrategyType]): """ _strategy = strategy_type(services) super().__init__(*self._make_pipeline_definitions(services, _strategy)) - self._services = list(services) + self._services = services self._strategy = _strategy @property @@ -251,7 +250,7 @@ class ServiceSwitcher(ParallelPipeline, Generic[StrategyType]): @staticmethod def _make_pipeline_definitions( - services: Sequence[FrameProcessor], strategy: ServiceSwitcherStrategy + services: list[FrameProcessor], strategy: ServiceSwitcherStrategy ) -> list[Any]: pipelines = [] for service in services: From 42a6fc703cbc8c773917b2ef941aca05d768c932 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 17 Apr 2026 15:38:14 -0400 Subject: [PATCH 15/49] Address review feedback - Fall back to Language.EN in _primary_detected_language when model is flux-general-en, preserving prior behavior on the default model. - Standardize example on DeepgramFluxSTTService.Settings and drop the now-redundant DeepgramFluxSTTSettings import. - Narrow the changed-behavior changelog to reflect that flux-general-en frames still carry Language.EN. --- changelog/4326.changed.md | 2 +- examples/update-settings/stt/stt-deepgram-flux.py | 7 ++++--- src/pipecat/services/deepgram/flux/base.py | 12 ++++++++---- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/changelog/4326.changed.md b/changelog/4326.changed.md index 2809777fc..4cc6e9bb7 100644 --- a/changelog/4326.changed.md +++ b/changelog/4326.changed.md @@ -1 +1 @@ -- `TranscriptionFrame.language` and `InterimTranscriptionFrame.language` emitted by `DeepgramFluxSTTService` now reflect the language Deepgram detected for each turn (read from the `languages` field on Flux's `TurnInfo` event) instead of a statically configured value. On `flux-general-multi` this means per-turn accuracy for downstream consumers (e.g. TTS voice selection). On `flux-general-en` the field is absent in Flux responses, so emitted frames now carry `language=None` instead of the previously hardcoded `Language.EN`. +- `TranscriptionFrame.language` and `InterimTranscriptionFrame.language` emitted by `DeepgramFluxSTTService` now reflect the language Deepgram detected for each turn (read from the `languages` field on Flux's `TurnInfo` event). On `flux-general-multi` this gives per-turn accuracy for downstream consumers (e.g. TTS voice selection). `flux-general-en` continues to emit `Language.EN`. diff --git a/examples/update-settings/stt/stt-deepgram-flux.py b/examples/update-settings/stt/stt-deepgram-flux.py index 82e60ce67..fc3a90f31 100644 --- a/examples/update-settings/stt/stt-deepgram-flux.py +++ b/examples/update-settings/stt/stt-deepgram-flux.py @@ -23,7 +23,6 @@ from pipecat.processors.aggregators.llm_response_universal import ( from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.deepgram.flux.base import DeepgramFluxSTTSettings from pipecat.services.deepgram.flux.stt import DeepgramFluxSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.transcriptions.language import Language @@ -118,7 +117,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info("Updating Deepgram Flux STT settings: eot_threshold, keyterm") await task.queue_frame( STTUpdateSettingsFrame( - delta=DeepgramFluxSTTSettings( + delta=DeepgramFluxSTTService.Settings( eot_threshold=0.8, keyterm=["Pipecat", "Deepgram"], ) @@ -130,7 +129,9 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): await asyncio.sleep(10) logger.info("Updating Deepgram Flux STT settings: language_hints=[es]") await task.queue_frame( - STTUpdateSettingsFrame(delta=DeepgramFluxSTTSettings(language_hints=[Language.ES])) + STTUpdateSettingsFrame( + delta=DeepgramFluxSTTService.Settings(language_hints=[Language.ES]) + ) ) @transport.event_handler("on_client_disconnected") diff --git a/src/pipecat/services/deepgram/flux/base.py b/src/pipecat/services/deepgram/flux/base.py index 089b5525a..c54cdda38 100644 --- a/src/pipecat/services/deepgram/flux/base.py +++ b/src/pipecat/services/deepgram/flux/base.py @@ -611,12 +611,16 @@ class DeepgramFluxSTTBase(STTService): def _primary_detected_language(self, data: dict[str, Any]) -> Language | None: """Extract the primary detected language from a TurnInfo payload. - Only populated by ``flux-general-multi``; returns ``None`` otherwise. + On ``flux-general-multi`` the language is read from TurnInfo's + ``languages`` field. On ``flux-general-en`` the field is absent, so we + fall back to ``Language.EN`` to match the model's fixed language. """ codes = data.get("languages") or [] - if not codes: - return None - return _code_to_pipecat_language(codes[0]) + if codes: + return _code_to_pipecat_language(codes[0]) + if self._settings.model == "flux-general-en": + return Language.EN + return None async def _handle_end_of_turn(self, transcript: str, data: dict[str, Any]): """Handle EndOfTurn events from Deepgram Flux. From ce9c214eec80e1fbc750cdad9a85d42947d13c10 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 17 Apr 2026 16:51:00 -0400 Subject: [PATCH 16/49] Silence krisp_audio import logs on auto-import The two logger.error lines in krisp_instance.py fired at module-load time whenever anything transitively imported it (e.g. pipecat.turns.user_start pulling in krisp_viva_ip_user_turn_start_strategy), producing noisy output for users who never asked for Krisp. Drop the log calls and raise a more informative ImportError that names the affected classes so direct importers still get clear guidance. --- src/pipecat/audio/krisp_instance.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/pipecat/audio/krisp_instance.py b/src/pipecat/audio/krisp_instance.py index 94e3dfe3f..afb5848ee 100644 --- a/src/pipecat/audio/krisp_instance.py +++ b/src/pipecat/audio/krisp_instance.py @@ -15,9 +15,11 @@ from loguru import logger try: import krisp_audio except ModuleNotFoundError as e: - logger.error(f"Exception: {e}") - logger.error("In order to use the Krisp instance, you need to install krisp_audio.") - raise ImportError(f"Missing module: {e}") from e + raise ImportError( + "krisp_audio is required for Krisp audio features. " + "Install it to use KrispVivaFilter, KrispVivaVadAnalyzer, " + "KrispVivaTurn, or KrispVivaIPUserTurnStartStrategy." + ) from e # Mapping of sample rates (Hz) to Krisp SDK SamplingRate enums From 74d11dc0aa6219554b07d49192f02215a430e56e Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sun, 19 Apr 2026 09:19:15 -0400 Subject: [PATCH 17/49] Silence pyright diagnostics for unchecked modules in IDE Pylance analyzes open files even when they're outside the `include` set, producing noise in the editor. Adding these paths to `ignore` suppresses diagnostics without affecting import resolution. --- pyrightconfig.json | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pyrightconfig.json b/pyrightconfig.json index 17359ed4a..6dae89b0a 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -17,5 +17,20 @@ "**/*_pb2.py", "**/__pycache__" ], + "ignore": [ + "src/pipecat/adapters", + "src/pipecat/audio", + "src/pipecat/processors", + "src/pipecat/serializers", + "src/pipecat/services", + "src/pipecat/sync", + "src/pipecat/tests", + "src/pipecat/transports", + "src/pipecat/utils", + "tests", + "examples", + "scripts", + "docs" + ], "reportMissingImports": false } From 995f897b80c6a34bbe0cd842fabd0d147131ff60 Mon Sep 17 00:00:00 2001 From: sathwika Date: Fri, 10 Apr 2026 17:58:06 +0530 Subject: [PATCH 18/49] Enhance NVIDIA LLM reasoning tokens handling and allow keyless local NIM endpoints --- src/pipecat/services/nvidia/llm.py | 222 ++++++++++++++++++++++++++--- 1 file changed, 204 insertions(+), 18 deletions(-) diff --git a/src/pipecat/services/nvidia/llm.py b/src/pipecat/services/nvidia/llm.py index 28b635a62..cb9cf275d 100644 --- a/src/pipecat/services/nvidia/llm.py +++ b/src/pipecat/services/nvidia/llm.py @@ -2,21 +2,38 @@ # Copyright (c) 2024-2026, Daily # # SPDX-License-Identifier: BSD 2-Clause License +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. # """NVIDIA NIM API service implementation. This module provides a service for interacting with NVIDIA's NIM (NVIDIA Inference Microservice) API while maintaining compatibility with the OpenAI-style interface. + +Refer to the NVIDIA NIM LLM API documentation for available models and usage: +https://docs.api.nvidia.com/nim/reference/llm-apis """ from dataclasses import dataclass +from typing import AsyncIterator, Optional +from loguru import logger +from openai import AsyncStream +from openai.types.chat import ChatCompletionChunk + +from pipecat.frames.frames import ( + LLMThoughtEndFrame, + LLMThoughtStartFrame, + LLMThoughtTextFrame, +) from pipecat.metrics.metrics import LLMTokenUsage from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.services.openai.base_llm import BaseOpenAILLMService from pipecat.services.openai.llm import OpenAILLMService +_THINK_OPEN = "" +_THINK_CLOSE = "" + @dataclass class NvidiaLLMSettings(BaseOpenAILLMService.Settings): @@ -28,9 +45,18 @@ class NvidiaLLMSettings(BaseOpenAILLMService.Settings): class NvidiaLLMService(OpenAILLMService): """A service for interacting with NVIDIA's NIM (NVIDIA Inference Microservice) API. - This service extends OpenAILLMService to work with NVIDIA's NIM API while maintaining - compatibility with the OpenAI-style interface. It specifically handles the difference - in token usage reporting between NIM (incremental) and OpenAI (final summary). + This service extends OpenAILLMService to work with NVIDIA's NIM API while + maintaining compatibility with the OpenAI-style interface. It handles: + + - Incremental token usage reporting (NIM sends per-chunk counts instead + of a final summary) + - Automatic detection and filtering of reasoning tokens from models that + emit ````/```` tags in content (e.g. DeepSeek-R1, some nemotron models) + - Extraction of ``reasoning_content`` from the streaming delta for models + with API-level reasoning separation (e.g. Nemotron Nano models) + + Reasoning content is emitted as ``LLMThought*Frame`` objects, keeping it + accessible to observers and logging without sending it to TTS. """ Settings = NvidiaLLMSettings @@ -39,7 +65,7 @@ class NvidiaLLMService(OpenAILLMService): def __init__( self, *, - api_key: str, + api_key: Optional[str] = None, base_url: str = "https://integrate.api.nvidia.com/v1", model: str | None = None, settings: Settings | None = None, @@ -48,10 +74,12 @@ class NvidiaLLMService(OpenAILLMService): """Initialize the NvidiaLLMService. Args: - api_key: The API key for accessing NVIDIA's NIM API. - base_url: The base URL for NIM API. Defaults to "https://integrate.api.nvidia.com/v1". + api_key: NVIDIA API key for authentication. Required when using the + cloud endpoint. Not needed for local NIM deployments. + base_url: The base URL for NIM API. Defaults to NVIDIA's cloud endpoint. + For local deployments, pass the local address (e.g. ``http://localhost:8000/v1``). model: The model identifier to use. Defaults to - "nvidia/llama-3.1-nemotron-70b-instruct". + "nvidia/nemotron-3-nano-30b-a3b". .. deprecated:: 0.0.105 Use ``settings=NvidiaLLMService.Settings(model=...)`` instead. @@ -61,7 +89,7 @@ class NvidiaLLMService(OpenAILLMService): **kwargs: Additional keyword arguments passed to OpenAILLMService. """ # 1. Initialize default_settings with hardcoded defaults - default_settings = self.Settings(model="nvidia/llama-3.1-nemotron-70b-instruct") + default_settings = self.Settings(model="nvidia/nemotron-3-nano-30b-a3b") # 2. Apply direct init arg overrides (deprecated) if model is not None: @@ -75,6 +103,14 @@ class NvidiaLLMService(OpenAILLMService): default_settings.apply_update(settings) super().__init__(api_key=api_key, base_url=base_url, settings=default_settings, **kwargs) + + if "api.nvidia.com" in base_url and not api_key: + logger.warning( + "NvidiaLLMService: Using the cloud endpoint but no API key was provided. " + "An API key is required for the cloud endpoint. " + "Set base_url to your local NIM endpoint for local deployments." + ) + # Counters for accumulating token usage metrics self._prompt_tokens = 0 self._completion_tokens = 0 @@ -82,26 +118,176 @@ class NvidiaLLMService(OpenAILLMService): self._has_reported_prompt_tokens = False self._is_processing = False - async def _process_context(self, context: LLMContext): - """Process a context through the LLM and accumulate token usage metrics. + def _reset_response_state(self): + """Reset per-response state at the start of each LLM call. - This method overrides the parent class implementation to handle NVIDIA's - incremental token reporting style, accumulating the counts and reporting - them once at the end of processing. - - Args: - context: The context to process, containing messages and other information - needed for the LLM interaction. + Resets token accumulation counters, thinking-tag detection state, + and reasoning-content field tracking. """ - # Reset all counters and flags at the start of processing self._prompt_tokens = 0 self._completion_tokens = 0 self._total_tokens = 0 self._has_reported_prompt_tokens = False self._is_processing = True + # tag detection: "detecting" → "in_thought" | "content" + self._think_tag_state = "detecting" + self._think_tag_buffer = "" + + # reasoning_content field tracking + self._has_reasoning_field = False + + async def _push_llm_text(self, text: str): + """Push LLM text, auto-detecting and filtering ```` tags. + + Uses a three-state machine to handle reasoning tokens in content: + + - ``detecting``: Buffers the first few chars to check for ````. + - ``in_thought``: Inside a think block; emits ``LLMThoughtTextFrame`` + until ```` is found. + - ``content``: Normal content; direct passthrough to base class. + + Non-reasoning models transition from ``detecting`` to ``content`` + on the first chunk with zero buffering overhead after that. + + Args: + text: The text content from the LLM to push. + """ + if self._think_tag_state == "content": + await super()._push_llm_text(text) + return + + self._think_tag_buffer += text + + if self._think_tag_state == "detecting": + if len(self._think_tag_buffer) < len(_THINK_OPEN): + if _THINK_OPEN.startswith(self._think_tag_buffer): + return + self._think_tag_state = "content" + await super()._push_llm_text(self._think_tag_buffer) + self._think_tag_buffer = "" + return + + if self._think_tag_buffer.startswith(_THINK_OPEN): + self._think_tag_state = "in_thought" + await self.push_frame(LLMThoughtStartFrame()) + self._think_tag_buffer = self._think_tag_buffer[len(_THINK_OPEN) :] + else: + self._think_tag_state = "content" + await super()._push_llm_text(self._think_tag_buffer) + self._think_tag_buffer = "" + return + + if self._think_tag_state == "in_thought": + idx = self._think_tag_buffer.find(_THINK_CLOSE) + if idx != -1: + thought = self._think_tag_buffer[:idx] + if thought: + await self.push_frame(LLMThoughtTextFrame(text=thought)) + await self.push_frame(LLMThoughtEndFrame()) + remainder = self._think_tag_buffer[idx + len(_THINK_CLOSE) :] + self._think_tag_buffer = "" + self._think_tag_state = "content" + if remainder: + await super()._push_llm_text(remainder) + else: + safe_end = len(self._think_tag_buffer) - len(_THINK_CLOSE) + 1 + if safe_end > 0: + await self.push_frame( + LLMThoughtTextFrame(text=self._think_tag_buffer[:safe_end]) + ) + self._think_tag_buffer = self._think_tag_buffer[safe_end:] + + async def get_chat_completions(self, context: LLMContext) -> AsyncIterator[ChatCompletionChunk]: + """Wrap the chat completion stream to handle ``reasoning_content``. + + Models with API-level reasoning separation (e.g. Nemotron Nano) + include a ``reasoning_content`` field on the streaming delta. This + wrapper extracts those chunks and emits them as ``LLMThought*Frame`` + objects, keeping them out of the normal content path. + + Args: + context: The LLM context for the completion request. + + Returns: + An async iterator of chat completion chunks where + ``reasoning_content`` has been emitted as ``LLMThought*Frame`` + side effects. + """ + stream = await super().get_chat_completions(context) + return self._handle_reasoning_content(stream) + + async def _handle_reasoning_content( + self, stream: AsyncStream[ChatCompletionChunk] + ) -> AsyncIterator[ChatCompletionChunk]: + """Handle ``reasoning_content`` from a chat completion chunk stream. + + Inspects each chunk for a ``reasoning_content`` field on the delta and + emits ``LLMThoughtStartFrame`` / ``LLMThoughtTextFrame`` / + ``LLMThoughtEndFrame`` as side effects. Every chunk (including + reasoning-only ones) is still yielded so the base streaming loop + can process metadata such as token usage and model name. + + Notes: + Stream cleanup is owned by the base OpenAI processing loop + (``BaseOpenAILLMService._process_context``), which wraps the stream + in its own closing context manager. + + Args: + stream: The original chat completion stream. + + Yields: + All chat completion chunks, unchanged. + """ + async for chunk in stream: + if chunk.choices and len(chunk.choices) > 0 and chunk.choices[0].delta: + rc = getattr(chunk.choices[0].delta, "reasoning_content", None) + if rc: + if not self._has_reasoning_field: + self._has_reasoning_field = True + await self.push_frame(LLMThoughtStartFrame()) + await self.push_frame(LLMThoughtTextFrame(text=rc)) + elif self._has_reasoning_field and chunk.choices[0].delta.content: + await self.push_frame(LLMThoughtEndFrame()) + self._has_reasoning_field = False + yield chunk + + async def _process_context(self, context: LLMContext): + """Process a context through the LLM and accumulate token usage metrics. + + Delegates to the base OpenAI streaming loop while adding + NVIDIA-specific behavior: + + - ``reasoning_content`` is intercepted via the + ``get_chat_completions`` stream wrapper and emitted as + ``LLMThought*Frame`` objects. + - ```` tag detection is handled by the ``_push_llm_text`` + override for models that embed reasoning in content. + - Incremental token counts are accumulated and reported as final + totals. + + Args: + context: The context to process, containing messages and other + information needed for the LLM interaction. + """ + self._reset_response_state() + + # Wrap in try/finally to guarantee accumulated token metrics are + # reported and _is_processing is cleared even on cancellation. try: await super()._process_context(context) + + # Flush any pending think-tag state (normal completion only; + # CancelledError skips this block). + if self._think_tag_state == "in_thought": + if self._think_tag_buffer: + await self.push_frame(LLMThoughtTextFrame(text=self._think_tag_buffer)) + await self.push_frame(LLMThoughtEndFrame()) + elif self._think_tag_buffer: + await super()._push_llm_text(self._think_tag_buffer) + + if self._has_reasoning_field: + await self.push_frame(LLMThoughtEndFrame()) finally: self._is_processing = False # Report final accumulated token usage at the end of processing From 74becffe55778848537d5b12892e44054ebdac0d Mon Sep 17 00:00:00 2001 From: sathwika Date: Fri, 10 Apr 2026 18:02:12 +0530 Subject: [PATCH 19/49] add changelog --- changelog/4270.changed.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/4270.changed.md diff --git a/changelog/4270.changed.md b/changelog/4270.changed.md new file mode 100644 index 000000000..eb9f54972 --- /dev/null +++ b/changelog/4270.changed.md @@ -0,0 +1 @@ +- Updated `NvidiaLLMService` to emit model reasoning as `LLMThought*Frame`s (from both `reasoning_content` and `...` output), avoid mixing reasoning text into normal assistant content, and allow keyless local NIM endpoints while warning when the cloud endpoint is used without an API key. From f2a19cb1a33b7f4307918149fbf5c394af06d488 Mon Sep 17 00:00:00 2001 From: dhruvladia-sarvam Date: Mon, 20 Apr 2026 13:52:48 +0530 Subject: [PATCH 20/49] Initial commit for vad parameters on saaras:v3 --- pyproject.toml | 2 +- src/pipecat/services/sarvam/stt.py | 167 ++++++++++++++++++++++++++++- 2 files changed, 165 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fca7db757..b1ff2cbd2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -106,7 +106,7 @@ rime = [ "pipecat-ai[websockets-base]" ] runner = [ "python-dotenv>=1.0.0,<2.0.0", "uvicorn>=0.32.0,<1.0.0", "fastapi>=0.115.6,<1", "pipecat-ai-small-webrtc-prebuilt>=2.4.0"] sagemaker = ["aws_sdk_sagemaker_runtime_http2; python_version>='3.12'"] sambanova = [] -sarvam = [ "sarvamai==0.1.26", "pipecat-ai[websockets-base]" ] +sarvam = [ "sarvamai==0.1.28", "pipecat-ai[websockets-base]" ] sentry = [ "sentry-sdk>=2.28.0,<3" ] silero = [] simli = [ "simli-ai~=2.0.1"] diff --git a/src/pipecat/services/sarvam/stt.py b/src/pipecat/services/sarvam/stt.py index faadd914b..4ebe703fa 100644 --- a/src/pipecat/services/sarvam/stt.py +++ b/src/pipecat/services/sarvam/stt.py @@ -91,6 +91,7 @@ class ModelConfig: supports_prompt: Whether the model accepts prompt parameter. supports_mode: Whether the model accepts mode parameter. supports_language: Whether the model accepts language parameter. + supports_vad_params: Whether the model accepts fine-grained VAD parameters. default_language: Default language code (None = auto-detect). default_mode: Default mode (None = not applicable). use_translate_endpoint: Whether to use speech_to_text_translate_streaming endpoint. @@ -100,6 +101,7 @@ class ModelConfig: supports_prompt: bool supports_mode: bool supports_language: bool + supports_vad_params: bool default_language: str | None default_mode: str | None use_translate_endpoint: bool @@ -111,6 +113,7 @@ MODEL_CONFIGS: dict[str, ModelConfig] = { supports_prompt=False, supports_mode=False, supports_language=True, + supports_vad_params=False, default_language="unknown", default_mode=None, use_translate_endpoint=False, @@ -120,6 +123,7 @@ MODEL_CONFIGS: dict[str, ModelConfig] = { supports_prompt=True, supports_mode=False, supports_language=False, + supports_vad_params=False, default_language=None, # Auto-detects language default_mode=None, use_translate_endpoint=True, @@ -129,6 +133,7 @@ MODEL_CONFIGS: dict[str, ModelConfig] = { supports_prompt=False, supports_mode=True, supports_language=True, + supports_vad_params=True, default_language="unknown", default_mode="transcribe", use_translate_endpoint=False, @@ -146,11 +151,43 @@ class SarvamSTTSettings(STTSettings): Only applicable to models that support prompts (e.g., saaras:v2.5). vad_signals: Enable VAD signals in response. high_vad_sensitivity: Enable high VAD sensitivity. + positive_speech_threshold: VAD probability threshold (0.0-1.0) above which + a frame is considered speech. Only for saaras:v3. + negative_speech_threshold: VAD probability threshold (0.0-1.0) below which + a frame is considered silence. Only for saaras:v3. + min_speech_frames: Minimum consecutive speech frames to start a speech + segment. Only for saaras:v3. + first_turn_min_speech_frames: Minimum speech frames for the first user + turn. Only for saaras:v3. + negative_frames_count: Number of silence frames within the window to end + a speech segment. Only for saaras:v3. + negative_frames_window: Sliding window size (in frames) for counting + negative frames. Only for saaras:v3. + start_speech_volume_threshold: Volume level (dB) below which audio is + too quiet to be speech. Only for saaras:v3. + interrupt_min_speech_frames: Minimum speech frames to register a + barge-in/interruption. Only for saaras:v3. + pre_speech_pad_frames: Number of audio frames to prepend before detected + speech onset. Only for saaras:v3. + num_initial_ignored_frames: Number of leading audio frames to skip at + connection start. Only for saaras:v3. """ prompt: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) vad_signals: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) high_vad_sensitivity: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + positive_speech_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + negative_speech_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + min_speech_frames: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + first_turn_min_speech_frames: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + negative_frames_count: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + negative_frames_window: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + start_speech_volume_threshold: float | None | _NotGiven = field( + default_factory=lambda: NOT_GIVEN + ) + interrupt_min_speech_frames: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + pre_speech_pad_frames: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + num_initial_ignored_frames: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) class SarvamSTTService(STTService): @@ -190,7 +227,27 @@ class SarvamSTTService(STTService): mode: Mode of operation for saaras:v3 models only. Options: transcribe, translate, verbatim, translit, codemix. Defaults to "transcribe" for saaras:v3. vad_signals: Enable VAD signals in response. Defaults to None. - high_vad_sensitivity: Enable high VAD (Voice Activity Detection) sensitivity. Defaults to None. + high_vad_sensitivity: Enable high VAD sensitivity. Defaults to None. + positive_speech_threshold: VAD probability threshold (0.0-1.0) above which + a frame is considered speech. Only for saaras:v3. + negative_speech_threshold: VAD probability threshold (0.0-1.0) below which + a frame is considered silence. Only for saaras:v3. + min_speech_frames: Minimum consecutive speech frames to start a segment. + Only for saaras:v3. + first_turn_min_speech_frames: Minimum speech frames for the first user + turn. Only for saaras:v3. + negative_frames_count: Silence frames within the window to end a segment. + Only for saaras:v3. + negative_frames_window: Sliding window size (in frames) for counting + negative frames. Only for saaras:v3. + start_speech_volume_threshold: Volume (dB) below which audio is too quiet. + Only for saaras:v3. + interrupt_min_speech_frames: Minimum speech frames for barge-in. + Only for saaras:v3. + pre_speech_pad_frames: Audio frames to prepend before speech onset. + Only for saaras:v3. + num_initial_ignored_frames: Leading frames to skip at connection start. + Only for saaras:v3. """ language: Language | None = None @@ -198,6 +255,16 @@ class SarvamSTTService(STTService): mode: Literal["transcribe", "translate", "verbatim", "translit", "codemix"] | None = None vad_signals: bool | None = None high_vad_sensitivity: bool | None = None + positive_speech_threshold: float | None = None + negative_speech_threshold: float | None = None + min_speech_frames: int | None = None + first_turn_min_speech_frames: int | None = None + negative_frames_count: int | None = None + negative_frames_window: int | None = None + start_speech_volume_threshold: float | None = None + interrupt_min_speech_frames: int | None = None + pre_speech_pad_frames: int | None = None + num_initial_ignored_frames: int | None = None def __init__( self, @@ -249,6 +316,16 @@ class SarvamSTTService(STTService): prompt=None, vad_signals=None, high_vad_sensitivity=None, + positive_speech_threshold=None, + negative_speech_threshold=None, + min_speech_frames=None, + first_turn_min_speech_frames=None, + negative_frames_count=None, + negative_frames_window=None, + start_speech_volume_threshold=None, + interrupt_min_speech_frames=None, + pre_speech_pad_frames=None, + num_initial_ignored_frames=None, ) # --- 2. Deprecated direct-arg overrides --- @@ -266,6 +343,18 @@ class SarvamSTTService(STTService): mode = params.mode default_settings.vad_signals = params.vad_signals default_settings.high_vad_sensitivity = params.high_vad_sensitivity + default_settings.positive_speech_threshold = params.positive_speech_threshold + default_settings.negative_speech_threshold = params.negative_speech_threshold + default_settings.min_speech_frames = params.min_speech_frames + default_settings.first_turn_min_speech_frames = params.first_turn_min_speech_frames + default_settings.negative_frames_count = params.negative_frames_count + default_settings.negative_frames_window = params.negative_frames_window + default_settings.start_speech_volume_threshold = ( + params.start_speech_volume_threshold + ) + default_settings.interrupt_min_speech_frames = params.interrupt_min_speech_frames + default_settings.pre_speech_pad_frames = params.pre_speech_pad_frames + default_settings.num_initial_ignored_frames = params.num_initial_ignored_frames # --- 4. Settings delta (canonical API, always wins) --- if settings is not None: @@ -289,6 +378,26 @@ class SarvamSTTService(STTService): f"Model '{resolved_model}' does not support language parameter (auto-detects language)." ) + if not self._config.supports_vad_params: + vad_param_names = [ + "positive_speech_threshold", + "negative_speech_threshold", + "min_speech_frames", + "first_turn_min_speech_frames", + "negative_frames_count", + "negative_frames_window", + "start_speech_volume_threshold", + "interrupt_min_speech_frames", + "pre_speech_pad_frames", + "num_initial_ignored_frames", + ] + for param_name in vad_param_names: + if getattr(default_settings, param_name) is not None: + raise ValueError( + f"Model '{resolved_model}' does not support {param_name} parameter. " + f"Fine-grained VAD parameters are only supported by saaras:v3." + ) + # Resolve mode default from model config if mode is None: mode = self._config.default_mode @@ -393,10 +502,44 @@ class SarvamSTTService(STTService): f"Model '{self._settings.model}' does not support prompt parameter." ) + if isinstance(delta, self.Settings) and not self._config.supports_vad_params: + vad_param_names = [ + "positive_speech_threshold", + "negative_speech_threshold", + "min_speech_frames", + "first_turn_min_speech_frames", + "negative_frames_count", + "negative_frames_window", + "start_speech_volume_threshold", + "interrupt_min_speech_frames", + "pre_speech_pad_frames", + "num_initial_ignored_frames", + ] + for param_name in vad_param_names: + val = getattr(delta, param_name, NOT_GIVEN) + if is_given(val) and val is not None: + raise ValueError( + f"Model '{self._settings.model}' does not support {param_name} " + f"parameter. Fine-grained VAD parameters are only supported by saaras:v3." + ) + changed = await super()._update_settings(delta) - # Language and prompt are WebSocket connect-time parameters; reconnect to apply. - reconnect_fields = {"language", "prompt"} + # These are all WebSocket connect-time parameters; reconnect to apply. + reconnect_fields = { + "language", + "prompt", + "positive_speech_threshold", + "negative_speech_threshold", + "min_speech_frames", + "first_turn_min_speech_frames", + "negative_frames_count", + "negative_frames_window", + "start_speech_volume_threshold", + "interrupt_min_speech_frames", + "pre_speech_pad_frames", + "num_initial_ignored_frames", + } if changed.keys() & reconnect_fields: await self._disconnect() await self._connect() @@ -535,6 +678,24 @@ class SarvamSTTService(STTService): "true" if self._settings.high_vad_sensitivity else "false" ) + # Fine-grained VAD parameters (saaras:v3 only, sent as strings per SDK spec) + if self._config.supports_vad_params: + _vad_params = { + "positive_speech_threshold": self._settings.positive_speech_threshold, + "negative_speech_threshold": self._settings.negative_speech_threshold, + "min_speech_frames": self._settings.min_speech_frames, + "first_turn_min_speech_frames": self._settings.first_turn_min_speech_frames, + "negative_frames_count": self._settings.negative_frames_count, + "negative_frames_window": self._settings.negative_frames_window, + "start_speech_volume_threshold": self._settings.start_speech_volume_threshold, + "interrupt_min_speech_frames": self._settings.interrupt_min_speech_frames, + "pre_speech_pad_frames": self._settings.pre_speech_pad_frames, + "num_initial_ignored_frames": self._settings.num_initial_ignored_frames, + } + for k, v in _vad_params.items(): + if v is not None: + connect_kwargs[k] = str(v) + # Add language_code for models that support it language_string = self._get_language_string() if language_string is not None: From 91e5b1ad9a75b9a69ec56fb63e1bef48bf4f435a Mon Sep 17 00:00:00 2001 From: sathwika Date: Mon, 20 Apr 2026 14:17:39 +0530 Subject: [PATCH 21/49] Handle NVIDIA LLM reasoning content in stream wrapper --- src/pipecat/services/nvidia/llm.py | 149 ++++++++++++++++++----------- 1 file changed, 92 insertions(+), 57 deletions(-) diff --git a/src/pipecat/services/nvidia/llm.py b/src/pipecat/services/nvidia/llm.py index cb9cf275d..7ddbe42e4 100644 --- a/src/pipecat/services/nvidia/llm.py +++ b/src/pipecat/services/nvidia/llm.py @@ -14,11 +14,11 @@ Refer to the NVIDIA NIM LLM API documentation for available models and usage: https://docs.api.nvidia.com/nim/reference/llm-apis """ +from collections.abc import AsyncIterator from dataclasses import dataclass -from typing import AsyncIterator, Optional +from enum import StrEnum from loguru import logger -from openai import AsyncStream from openai.types.chat import ChatCompletionChunk from pipecat.frames.frames import ( @@ -35,6 +35,12 @@ _THINK_OPEN = "" _THINK_CLOSE = "" +class _ThinkTagState(StrEnum): + DETECTING = "detecting" + IN_THOUGHT = "in_thought" + CONTENT = "content" + + @dataclass class NvidiaLLMSettings(BaseOpenAILLMService.Settings): """Settings for NvidiaLLMService.""" @@ -50,8 +56,9 @@ class NvidiaLLMService(OpenAILLMService): - Incremental token usage reporting (NIM sends per-chunk counts instead of a final summary) - - Automatic detection and filtering of reasoning tokens from models that - emit ````/```` tags in content (e.g. DeepSeek-R1, some nemotron models) + - Detection and filtering of leading ````/```` content for + models that emit reasoning inline before visible output (e.g. + DeepSeek-R1, some nemotron models) - Extraction of ``reasoning_content`` from the streaming delta for models with API-level reasoning separation (e.g. Nemotron Nano models) @@ -65,7 +72,7 @@ class NvidiaLLMService(OpenAILLMService): def __init__( self, *, - api_key: Optional[str] = None, + api_key: str | None = None, base_url: str = "https://integrate.api.nvidia.com/v1", model: str | None = None, settings: Settings | None = None, @@ -121,7 +128,7 @@ class NvidiaLLMService(OpenAILLMService): def _reset_response_state(self): """Reset per-response state at the start of each LLM call. - Resets token accumulation counters, thinking-tag detection state, + Resets token accumulation counters, leading-think-tag detection state, and reasoning-content field tracking. """ self._prompt_tokens = 0 @@ -130,55 +137,64 @@ class NvidiaLLMService(OpenAILLMService): self._has_reported_prompt_tokens = False self._is_processing = True - # tag detection: "detecting" → "in_thought" | "content" - self._think_tag_state = "detecting" + self._think_tag_state = _ThinkTagState.DETECTING self._think_tag_buffer = "" # reasoning_content field tracking self._has_reasoning_field = False - async def _push_llm_text(self, text: str): - """Push LLM text, auto-detecting and filtering ```` tags. + async def _filter_thinking_content(self, text: str) -> str | None: + """Filter leading ```` tags from content and emit thought frames. - Uses a three-state machine to handle reasoning tokens in content: + Uses a three-state machine optimized for the common provider pattern + where a response either begins with a ```` block or contains no + think tags at all. It returns only visible content to the base OpenAI + processing loop while emitting hidden reasoning as ``LLMThought*Frame`` + side effects. - - ``detecting``: Buffers the first few chars to check for ````. - - ``in_thought``: Inside a think block; emits ``LLMThoughtTextFrame`` - until ```` is found. - - ``content``: Normal content; direct passthrough to base class. + - ``detecting``: Buffers the start of the stream to check for + ````. + - ``in_thought``: Inside a leading think block; emits + ``LLMThoughtTextFrame`` until ```` is found. + - ``content``: Normal content; passthrough. Non-reasoning models transition from ``detecting`` to ``content`` on the first chunk with zero buffering overhead after that. Args: - text: The text content from the LLM to push. + text: The text content from the LLM to filter. + + Returns: + The non-reasoning content that should continue through the base + OpenAI content path, or ``None`` if this chunk should not emit + normal content. + """ - if self._think_tag_state == "content": - await super()._push_llm_text(text) - return + if self._think_tag_state == _ThinkTagState.CONTENT: + return text self._think_tag_buffer += text - if self._think_tag_state == "detecting": + if self._think_tag_state == _ThinkTagState.DETECTING: if len(self._think_tag_buffer) < len(_THINK_OPEN): if _THINK_OPEN.startswith(self._think_tag_buffer): - return - self._think_tag_state = "content" - await super()._push_llm_text(self._think_tag_buffer) + return None + self._think_tag_state = _ThinkTagState.CONTENT + passthrough = self._think_tag_buffer self._think_tag_buffer = "" - return + return passthrough if self._think_tag_buffer.startswith(_THINK_OPEN): - self._think_tag_state = "in_thought" + self._think_tag_state = _ThinkTagState.IN_THOUGHT await self.push_frame(LLMThoughtStartFrame()) self._think_tag_buffer = self._think_tag_buffer[len(_THINK_OPEN) :] else: - self._think_tag_state = "content" - await super()._push_llm_text(self._think_tag_buffer) + self._think_tag_state = _ThinkTagState.CONTENT + passthrough = self._think_tag_buffer self._think_tag_buffer = "" - return + return passthrough - if self._think_tag_state == "in_thought": + if self._think_tag_state == _ThinkTagState.IN_THOUGHT: idx = self._think_tag_buffer.find(_THINK_CLOSE) if idx != -1: thought = self._think_tag_buffer[:idx] @@ -187,9 +203,8 @@ class NvidiaLLMService(OpenAILLMService): await self.push_frame(LLMThoughtEndFrame()) remainder = self._think_tag_buffer[idx + len(_THINK_CLOSE) :] self._think_tag_buffer = "" - self._think_tag_state = "content" - if remainder: - await super()._push_llm_text(remainder) + self._think_tag_state = _ThinkTagState.CONTENT + return remainder or None else: safe_end = len(self._think_tag_buffer) - len(_THINK_CLOSE) + 1 if safe_end > 0: @@ -197,6 +212,28 @@ class NvidiaLLMService(OpenAILLMService): LLMThoughtTextFrame(text=self._think_tag_buffer[:safe_end]) ) self._think_tag_buffer = self._think_tag_buffer[safe_end:] + return None + + async def _flush_reasoning_state(self): + """Flush buffered reasoning state at normal stream completion. + + Emits any buffered trailing thought text, closes open thought frames, + and forwards any buffered pre-content text that was held while deciding + whether the stream began with ````. + """ + if self._think_tag_state == _ThinkTagState.IN_THOUGHT: + if self._think_tag_buffer: + await self.push_frame(LLMThoughtTextFrame(text=self._think_tag_buffer)) + await self.push_frame(LLMThoughtEndFrame()) + elif self._think_tag_state == _ThinkTagState.DETECTING and self._think_tag_buffer: + await super()._push_llm_text(self._think_tag_buffer) + + self._think_tag_buffer = "" + self._think_tag_state = _ThinkTagState.CONTENT + + if self._has_reasoning_field: + await self.push_frame(LLMThoughtEndFrame()) + self._has_reasoning_field = False async def get_chat_completions(self, context: LLMContext) -> AsyncIterator[ChatCompletionChunk]: """Wrap the chat completion stream to handle ``reasoning_content``. @@ -204,7 +241,9 @@ class NvidiaLLMService(OpenAILLMService): Models with API-level reasoning separation (e.g. Nemotron Nano) include a ``reasoning_content`` field on the streaming delta. This wrapper extracts those chunks and emits them as ``LLMThought*Frame`` - objects, keeping them out of the normal content path. + objects. It also rewrites streamed ``delta.content`` so leading + ```` sections are removed before the base OpenAI loop processes + visible content. Args: context: The LLM context for the completion request. @@ -218,15 +257,17 @@ class NvidiaLLMService(OpenAILLMService): return self._handle_reasoning_content(stream) async def _handle_reasoning_content( - self, stream: AsyncStream[ChatCompletionChunk] + self, stream: AsyncIterator[ChatCompletionChunk] ) -> AsyncIterator[ChatCompletionChunk]: - """Handle ``reasoning_content`` from a chat completion chunk stream. + """Handle ``reasoning_content`` and leading ```` tags in a chunk stream. Inspects each chunk for a ``reasoning_content`` field on the delta and emits ``LLMThoughtStartFrame`` / ``LLMThoughtTextFrame`` / - ``LLMThoughtEndFrame`` as side effects. Every chunk (including - reasoning-only ones) is still yielded so the base streaming loop - can process metadata such as token usage and model name. + ``LLMThoughtEndFrame`` as side effects. It also strips ```` + blocks from ``delta.content`` before yielding the chunk so the base + OpenAI loop only sees user-facing content. Every chunk is still yielded + so the base streaming loop can process metadata such as token usage, + model name, tool calls, and audio transcripts. Notes: Stream cleanup is owned by the base OpenAI processing loop @@ -237,32 +278,38 @@ class NvidiaLLMService(OpenAILLMService): stream: The original chat completion stream. Yields: - All chat completion chunks, unchanged. + Chat completion chunks with any leading ```` content removed + from ``delta.content`` before they reach the base OpenAI loop. """ async for chunk in stream: if chunk.choices and len(chunk.choices) > 0 and chunk.choices[0].delta: - rc = getattr(chunk.choices[0].delta, "reasoning_content", None) + delta = chunk.choices[0].delta + rc = getattr(delta, "reasoning_content", None) if rc: if not self._has_reasoning_field: self._has_reasoning_field = True await self.push_frame(LLMThoughtStartFrame()) await self.push_frame(LLMThoughtTextFrame(text=rc)) - elif self._has_reasoning_field and chunk.choices[0].delta.content: + elif self._has_reasoning_field and delta.content: await self.push_frame(LLMThoughtEndFrame()) self._has_reasoning_field = False + + if delta.content: + delta.content = await self._filter_thinking_content(delta.content) yield chunk + await self._flush_reasoning_state() + async def _process_context(self, context: LLMContext): """Process a context through the LLM and accumulate token usage metrics. Delegates to the base OpenAI streaming loop while adding NVIDIA-specific behavior: - - ``reasoning_content`` is intercepted via the - ``get_chat_completions`` stream wrapper and emitted as + - ``reasoning_content`` and leading ```` content are + intercepted via the ``get_chat_completions`` stream wrapper and + emitted as ``LLMThought*Frame`` objects. - - ```` tag detection is handled by the ``_push_llm_text`` - override for models that embed reasoning in content. - Incremental token counts are accumulated and reported as final totals. @@ -276,18 +323,6 @@ class NvidiaLLMService(OpenAILLMService): # reported and _is_processing is cleared even on cancellation. try: await super()._process_context(context) - - # Flush any pending think-tag state (normal completion only; - # CancelledError skips this block). - if self._think_tag_state == "in_thought": - if self._think_tag_buffer: - await self.push_frame(LLMThoughtTextFrame(text=self._think_tag_buffer)) - await self.push_frame(LLMThoughtEndFrame()) - elif self._think_tag_buffer: - await super()._push_llm_text(self._think_tag_buffer) - - if self._has_reasoning_field: - await self.push_frame(LLMThoughtEndFrame()) finally: self._is_processing = False # Report final accumulated token usage at the end of processing From 03bd667f95e11b25c35c9fff4d47f1ccbe81e08e Mon Sep 17 00:00:00 2001 From: Harshita Jain <77115160+harshitajain165@users.noreply.github.com> Date: Mon, 20 Apr 2026 20:45:25 +0530 Subject: [PATCH 22/49] Fix Smallest AI TTS WebSocket endpoint URL and remove unsupported flush (#4320) * Fix Smallest AI TTS WebSocket endpoint URL to match API documentation Update base URL from waves-api.smallest.ai to api.smallest.ai and fix path prefix from /api/v1/ to /waves/v1/ per the v4.0.0 docs. * Update keepalive using silent space message instead of unsupported flush --- changelog/4320.fixed.md | 1 + src/pipecat/services/smallest/tts.py | 22 ++++++++++++---------- 2 files changed, 13 insertions(+), 10 deletions(-) create mode 100644 changelog/4320.fixed.md diff --git a/changelog/4320.fixed.md b/changelog/4320.fixed.md new file mode 100644 index 000000000..12e49c957 --- /dev/null +++ b/changelog/4320.fixed.md @@ -0,0 +1 @@ +- Fixed `SmallestTTSService` WebSocket endpoint URL to match Smallest AI v4.0.0 API (`wss://waves-api.smallest.ai` → `wss://api.smallest.ai`) and restored keepalive using a silent space message instead of the unsupported flush command. diff --git a/src/pipecat/services/smallest/tts.py b/src/pipecat/services/smallest/tts.py index 33c492c01..8bc22035c 100644 --- a/src/pipecat/services/smallest/tts.py +++ b/src/pipecat/services/smallest/tts.py @@ -125,7 +125,7 @@ class SmallestTTSService(InterruptibleTTSService): self, *, api_key: str, - base_url: str = "wss://waves-api.smallest.ai", + base_url: str = "wss://api.smallest.ai", sample_rate: int | None = None, settings: Settings | None = None, **kwargs, @@ -174,6 +174,10 @@ class SmallestTTSService(InterruptibleTTSService): """ return True + async def flush_audio(self, context_id: str | None = None): + """Flush any pending audio data.""" + logger.trace(f"{self}: flushing audio") + def language_to_service_language(self, language: Language) -> str | None: """Convert a Language enum to Smallest service language format. @@ -217,7 +221,7 @@ class SmallestTTSService(InterruptibleTTSService): def _build_websocket_url(self) -> str: """Build the WebSocket URL from base URL and model.""" - return f"{self._base_url}/api/v1/{self._settings.model}/get_speech/stream" + return f"{self._base_url}/waves/v1/{self._settings.model}/get_speech/stream" async def start(self, frame: StartFrame): """Start the Smallest TTS service. @@ -350,17 +354,15 @@ class SmallestTTSService(InterruptibleTTSService): await self._send_keepalive() async def _send_keepalive(self): - """Send a flush message to keep the connection alive.""" + """Send a silent message to keep the WebSocket connection alive.""" if self._websocket and self._websocket.state is State.OPEN: - msg = {"flush": True} + msg = { + "text": " ", + "voice_id": self._settings.voice, + "language": self._settings.language, + } await self._websocket.send(json.dumps(msg)) - async def flush_audio(self, context_id: str | None = None): - """Flush any pending audio synthesis.""" - if not self._websocket or self._websocket.state is State.CLOSED: - return - await self._get_websocket().send(json.dumps({"flush": True})) - async def _receive_messages(self): """Receive and process messages from the Smallest WebSocket API.""" async for message in self._get_websocket(): From b59c4775da06625cec3bb864b07cd986225c177d Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 20 Apr 2026 11:55:09 -0400 Subject: [PATCH 23/49] Split user-turn stop timeout into independent speech and STT timers SpeechTimeoutUserTurnStopStrategy previously collapsed two waits into max(stt_timeout, user_speech_timeout), which over-waited for finalizing STT services and could also end the turn early in a legacy code path. Run them as independent timers instead: - user_speech_timeout: policy floor, always runs to completion. - stt_timeout: latency safety net, short-circuited by a finalized transcript since STT has signaled it has nothing more to send. The no-VAD fallback now waits only user_speech_timeout rather than max(stt_timeout, user_speech_timeout); stt_timeout is defined relative to VAD stop and has no meaning when no VAD event occurred. This shortens the fallback wait for users who set stt_timeout greater than user_speech_timeout. --- .../speech_timeout_user_turn_stop_strategy.py | 219 +++++++++++------- tests/test_user_turn_stop_strategy.py | 101 ++++++++ 2 files changed, 237 insertions(+), 83 deletions(-) diff --git a/src/pipecat/turns/user_stop/speech_timeout_user_turn_stop_strategy.py b/src/pipecat/turns/user_stop/speech_timeout_user_turn_stop_strategy.py index e37b55756..9a4060f1c 100644 --- a/src/pipecat/turns/user_stop/speech_timeout_user_turn_stop_strategy.py +++ b/src/pipecat/turns/user_stop/speech_timeout_user_turn_stop_strategy.py @@ -7,7 +7,6 @@ """Speech timeout-based user turn stop strategy.""" import asyncio -import time from loguru import logger @@ -25,20 +24,24 @@ from pipecat.utils.asyncio.task_manager import BaseTaskManager class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): - """User turn stop strategy that uses a configurable timeout to determine if the user is done speaking. + """User turn stop strategy using two independent timers after VAD stop. - After the user stops speaking (detected by VAD), this strategy waits for a - configurable timeout before triggering the end of the user's turn. The - timeout accounts for two factors: + After the user stops speaking (detected by VAD), this strategy runs two + independent timers. The user turn stop is triggered only when both have + finished and at least one transcript has been received: - - user_speech_timeout: Time to wait for the user to potentially say more - after they pause. - - stt_timeout: The P99 time for the STT service to return a transcription - after the user stops speaking, adjusted by the VAD stop_secs. + - user_speech_timeout: Policy floor — the window in which the user may + resume speaking after a pause. Always runs to completion. + - stt_timeout: Safety net for STT latency — the P99 time for the STT + service to return a final transcript after VAD stop, adjusted by the + VAD stop_secs. Short-circuited when the STT service emits a finalized + transcript (TranscriptionFrame.finalized=True), since finalization + means STT has nothing more to send. - For services that support finalization (TranscriptionFrame.finalized=True), - the turn can be triggered immediately once the finalized transcript is - received and the user resume speaking timeout has elapsed. + Fallback: when a transcript arrives without a VAD stop event, the + strategy waits only user_speech_timeout for inactivity (rearmed on each + transcript). stt_timeout has no meaning here since it is defined + relative to VAD stop, and STT has already emitted a transcript. """ def __init__(self, *, user_speech_timeout: float = 0.6, **kwargs): @@ -59,8 +62,16 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): self._vad_user_speaking = False self._transcript_finalized = False self._vad_stopped_time: float | None = None - self._timeout_task: asyncio.Task | None = None - self._timeout_expired: bool = False + + # VAD-driven timers and completion flags. + self._user_speech_timeout_task: asyncio.Task | None = None + self._stt_timeout_task: asyncio.Task | None = None + self._user_speech_expired: bool = False + self._stt_wait_done: bool = False + + # Fallback timer (transcript arrived without VAD stop). + self._fallback_timeout_task: asyncio.Task | None = None + self._fallback_expired: bool = False async def reset(self): """Reset the strategy to its initial state.""" @@ -69,10 +80,10 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): self._vad_user_speaking = False self._transcript_finalized = False self._vad_stopped_time = None - self._timeout_expired = False - if self._timeout_task: - await self.task_manager.cancel_task(self._timeout_task) - self._timeout_task = None + self._user_speech_expired = False + self._stt_wait_done = False + self._fallback_expired = False + await self._cancel_all_tasks() async def setup(self, task_manager: BaseTaskManager): """Initialize the strategy with the given task manager. @@ -85,9 +96,7 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): async def cleanup(self): """Cleanup the strategy.""" await super().cleanup() - if self._timeout_task: - await self.task_manager.cancel_task(self._timeout_task) - self._timeout_task = None + await self._cancel_all_tasks() async def process_frame(self, frame: Frame) -> ProcessFrameResult: """Process an incoming frame to update strategy state. @@ -105,8 +114,10 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): self._stt_timeout = frame.ttfs_p99_latency self._stop_secs_warned = False elif isinstance(frame, VADUserStartedSpeakingFrame): + logger.debug(f"{self} VADUserStartedSpeakingFrame received") await self._handle_vad_user_started_speaking(frame) elif isinstance(frame, VADUserStoppedSpeakingFrame): + logger.debug(f"{self} VADUserStoppedSpeakingFrame received") await self._handle_vad_user_stopped_speaking(frame) elif isinstance(frame, TranscriptionFrame): await self._handle_transcription(frame) @@ -118,11 +129,10 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): self._vad_user_speaking = True self._transcript_finalized = False self._vad_stopped_time = None - self._timeout_expired = False - # Cancel any pending timeout - if self._timeout_task: - await self.task_manager.cancel_task(self._timeout_task) - self._timeout_task = None + self._user_speech_expired = False + self._stt_wait_done = False + self._fallback_expired = False + await self._cancel_all_tasks() async def _handle_vad_user_stopped_speaking(self, frame: VADUserStoppedSpeakingFrame): """Handle when the VAD indicates the user has stopped speaking.""" @@ -150,59 +160,66 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): f"user_turn_stop_timeout parameter in the LLMUserAggregatorParams." ) - # Start the timeout task - timeout = self._calculate_timeout() - self._timeout_task = self.task_manager.create_task( - self._timeout_handler(timeout), f"{self}::_timeout_handler" + # Any prior fallback timer is superseded by the VAD-driven path. + if self._fallback_timeout_task: + await self.task_manager.cancel_task(self._fallback_timeout_task) + self._fallback_timeout_task = None + self._fallback_expired = False + + # user_speech_timeout is the policy floor and always runs. + self._user_speech_timeout_task = self.task_manager.create_task( + self._user_speech_timeout_handler(self._user_speech_timeout), + f"{self}::_user_speech_timeout_handler", ) - # Make sure the task is scheduled. + + # stt_timeout is a safety net. Short-circuit it if the transcript is + # already finalized, or if the VAD stop_secs already covered it. + effective_stt_wait = max(0.0, self._stt_timeout - self._stop_secs) + if self._transcript_finalized or effective_stt_wait <= 0: + self._stt_wait_done = True + else: + self._stt_timeout_task = self.task_manager.create_task( + self._stt_timeout_handler(effective_stt_wait), + f"{self}::_stt_timeout_handler", + ) + + # Make sure the tasks are scheduled. await asyncio.sleep(0) async def _handle_transcription(self, frame: TranscriptionFrame): """Handle user transcription.""" self._text += frame.text + if frame.finalized: self._transcript_finalized = True - # For finalized transcripts, check if we can trigger early + # Short-circuit the stt_timeout safety net: STT has told us + # there's nothing more coming. + if not self._stt_wait_done: + self._stt_wait_done = True + if self._stt_timeout_task: + await self.task_manager.cancel_task(self._stt_timeout_task) + self._stt_timeout_task = None + + # If both VAD-path timers are done (or the fallback timer already + # expired), the turn was waiting on text — trigger now. + if self._fallback_expired or (self._user_speech_expired and self._stt_wait_done): await self._maybe_trigger_user_turn_stopped() - elif self._timeout_expired: - # The p99 timeout already elapsed without a transcript. Now that - # we have one, trigger the turn stop immediately. - await self.trigger_user_turn_stopped() return # Fallback: handle transcripts when no VAD stop was received. - # This handles edge cases where transcripts arrive without VAD firing. - # _vad_stopped_time is None means VAD stopped hasn't been received yet. - # In fallback mode, reset timeout on each transcript to wait for inactivity. + # Rearm the fallback timer on each transcript to wait for inactivity. if not self._vad_user_speaking and self._vad_stopped_time is None: - # Cancel existing fallback timeout if any - if self._timeout_task: - await self.task_manager.cancel_task(self._timeout_task) - timeout = self._calculate_timeout() - self._timeout_task = self.task_manager.create_task( - self._timeout_handler(timeout), f"{self}::_timeout_handler" + if self._fallback_timeout_task: + await self.task_manager.cancel_task(self._fallback_timeout_task) + self._fallback_timeout_task = self.task_manager.create_task( + self._fallback_timeout_handler(self._user_speech_timeout), + f"{self}::_fallback_timeout_handler", ) # Make sure the task is scheduled. await asyncio.sleep(0) - def _calculate_timeout(self) -> float: - """Calculate the timeout value based on current state. - - Returns: - The timeout in seconds to wait after VAD stopped speaking. - """ - # Adjust STT timeout by VAD stop_secs since that time has already elapsed - effective_stt_wait = max(0, self._stt_timeout - self._stop_secs) - - # If transcript is already finalized, we don't need to wait for STT - if self._transcript_finalized: - return self._user_speech_timeout - - return max(effective_stt_wait, self._user_speech_timeout) - - async def _timeout_handler(self, timeout: float): - """Wait for the timeout then trigger user turn stopped if conditions met. + async def _user_speech_timeout_handler(self, timeout: float): + """Wait user_speech_timeout then attempt to trigger user turn stopped. Args: timeout: The timeout in seconds to wait. @@ -212,36 +229,72 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): except asyncio.CancelledError: return finally: - self._timeout_task = None + self._user_speech_timeout_task = None - self._timeout_expired = True + self._user_speech_expired = True + await self._maybe_trigger_user_turn_stopped() + + async def _stt_timeout_handler(self, timeout: float): + """Wait stt_timeout then attempt to trigger user turn stopped. + + Args: + timeout: The timeout in seconds to wait. + """ + try: + await asyncio.sleep(timeout) + except asyncio.CancelledError: + return + finally: + self._stt_timeout_task = None + + self._stt_wait_done = True + await self._maybe_trigger_user_turn_stopped() + + async def _fallback_timeout_handler(self, timeout: float): + """Wait user_speech_timeout of inactivity on the fallback path. + + Args: + timeout: The timeout in seconds to wait. + """ + try: + await asyncio.sleep(timeout) + except asyncio.CancelledError: + return + finally: + self._fallback_timeout_task = None + + self._fallback_expired = True await self._maybe_trigger_user_turn_stopped() async def _maybe_trigger_user_turn_stopped(self): - """Trigger user turn stopped if conditions are met. + """Trigger user turn stopped if all required conditions are met. - Conditions: - - User is not currently speaking - - We have transcription text - - Either the timeout has elapsed OR we have a finalized transcript - and user_speech_timeout has elapsed + VAD path: both user_speech_timeout and stt_timeout must have + completed (stt short-circuited by finalization counts as complete). + Fallback path: the fallback timer must have completed. + + In all cases, the user must not be currently speaking and at least + one transcript must have been received. """ if self._vad_user_speaking or not self._text: return - # For finalized transcripts, check if user_speech_timeout has elapsed. - # If elapsed, trigger user turn stopped immediately. Else, wait for user resume - # speaking timeout. - if self._transcript_finalized and self._vad_stopped_time is not None: - elapsed = time.time() - self._vad_stopped_time - if elapsed >= self._user_speech_timeout: - # Cancel any remaining timeout since we're triggering now - if self._timeout_task: - await self.task_manager.cancel_task(self._timeout_task) - self._timeout_task = None + if self._vad_stopped_time is not None: + if self._user_speech_expired and self._stt_wait_done: await self.trigger_user_turn_stopped() - return + return - # For non-finalized, only trigger if timeout task has completed - if self._timeout_task is None: + if self._fallback_expired: await self.trigger_user_turn_stopped() + + async def _cancel_all_tasks(self): + """Cancel any running timer tasks and clear the handles.""" + if self._user_speech_timeout_task: + await self.task_manager.cancel_task(self._user_speech_timeout_task) + self._user_speech_timeout_task = None + if self._stt_timeout_task: + await self.task_manager.cancel_task(self._stt_timeout_task) + self._stt_timeout_task = None + if self._fallback_timeout_task: + await self.task_manager.cancel_task(self._fallback_timeout_task) + self._fallback_timeout_task = None diff --git a/tests/test_user_turn_stop_strategy.py b/tests/test_user_turn_stop_strategy.py index ecaf5317f..9acceaf0f 100644 --- a/tests/test_user_turn_stop_strategy.py +++ b/tests/test_user_turn_stop_strategy.py @@ -529,6 +529,107 @@ class TestSpeechTimeoutUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase): # Non-finalized transcript received after timeout, triggers immediately self.assertTrue(should_start) + async def test_finalized_short_circuits_stt_wait(self): + """Finalized transcript cancels the stt_timeout safety net. + + user_speech_timeout still runs to completion as a policy floor, + but stt_timeout is skipped once STT says it's done. Net effect: + the turn stops at user_speech_timeout, not stt_timeout. + """ + stt_timeout = AGGREGATION_TIMEOUT * 4 + strategy = SpeechTimeoutUserTurnStopStrategy(user_speech_timeout=AGGREGATION_TIMEOUT) + await strategy.setup(self.task_manager) + await strategy.process_frame( + STTMetadataFrame(service_name="test", ttfs_p99_latency=stt_timeout) + ) + + should_start = None + + @strategy.event_handler("on_user_turn_stopped") + async def on_user_turn_stopped(strategy, params): + nonlocal should_start + should_start = True + + # S → E: starts user_speech_timeout (short) and stt_timeout (long). + await strategy.process_frame(VADUserStartedSpeakingFrame()) + await strategy.process_frame(VADUserStoppedSpeakingFrame()) + + # Finalized transcript arrives before user_speech_timeout elapses. + await strategy.process_frame( + TranscriptionFrame(text="Hello!", user_id="cat", timestamp="", finalized=True) + ) + # user_speech_timeout is still running, so no trigger yet. + self.assertIsNone(should_start) + + # user_speech_timeout elapses — stt_timeout was short-circuited, + # so the turn stops now rather than waiting for stt_timeout. + await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1) + self.assertTrue(should_start) + + async def test_non_finalized_waits_full_stt_timeout(self): + """Non-finalized transcript does not short-circuit stt_timeout. + + When STT never signals finalization, the stt_timeout safety net + must run its full course — the turn should not stop until the + longer of the two timers has elapsed. + """ + stt_timeout = AGGREGATION_TIMEOUT * 4 + strategy = SpeechTimeoutUserTurnStopStrategy(user_speech_timeout=AGGREGATION_TIMEOUT) + await strategy.setup(self.task_manager) + await strategy.process_frame( + STTMetadataFrame(service_name="test", ttfs_p99_latency=stt_timeout) + ) + + should_start = None + + @strategy.event_handler("on_user_turn_stopped") + async def on_user_turn_stopped(strategy, params): + nonlocal should_start + should_start = True + + # S → E: both timers start. + await strategy.process_frame(VADUserStartedSpeakingFrame()) + await strategy.process_frame(VADUserStoppedSpeakingFrame()) + + # Non-finalized transcript during the wait. + await strategy.process_frame(TranscriptionFrame(text="Hello!", user_id="cat", timestamp="")) + + # user_speech_timeout elapses but stt_timeout has not — no trigger. + await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1) + self.assertIsNone(should_start) + + # Wait for the remainder of stt_timeout. + await asyncio.sleep(stt_timeout - AGGREGATION_TIMEOUT + 0.1) + self.assertTrue(should_start) + + async def test_fallback_uses_only_user_speech_timeout(self): + """Fallback path (no VAD) ignores stt_timeout and uses only user_speech_timeout. + + stt_timeout is defined as "p99 after VAD stop" — without a VAD + reference point it has no meaning. The fallback measures + inactivity since the last transcript, which is user_speech_timeout. + """ + stt_timeout = AGGREGATION_TIMEOUT * 4 + strategy = SpeechTimeoutUserTurnStopStrategy(user_speech_timeout=AGGREGATION_TIMEOUT) + await strategy.setup(self.task_manager) + await strategy.process_frame( + STTMetadataFrame(service_name="test", ttfs_p99_latency=stt_timeout) + ) + + should_start = None + + @strategy.event_handler("on_user_turn_stopped") + async def on_user_turn_stopped(strategy, params): + nonlocal should_start + should_start = True + + # Transcript arrives without any VAD frame — fallback path. + await strategy.process_frame(TranscriptionFrame(text="Hello!", user_id="cat", timestamp="")) + + # The fallback timer is user_speech_timeout, not stt_timeout. + await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1) + self.assertTrue(should_start) + async def test_reset_clears_stale_text_no_premature_stop(self): """Test that reset() clears stale text and cancels timeout, preventing premature stop. From 9d8eefd2a231b09dfbea4456f1c00c1d0fa57f4d Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 20 Apr 2026 11:58:20 -0400 Subject: [PATCH 24/49] Add changelog for #4337 --- changelog/4337.changed.2.md | 1 + changelog/4337.changed.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog/4337.changed.2.md create mode 100644 changelog/4337.changed.md diff --git a/changelog/4337.changed.2.md b/changelog/4337.changed.2.md new file mode 100644 index 000000000..a1da061c8 --- /dev/null +++ b/changelog/4337.changed.2.md @@ -0,0 +1 @@ +- `SpeechTimeoutUserTurnStopStrategy` now waits only `user_speech_timeout` when a transcript arrives without a VAD stop event, rather than `max(ttfs_p99_latency, user_speech_timeout)`. If you had `ttfs_p99_latency > user_speech_timeout`, turn detection in that path is slightly faster than before. diff --git a/changelog/4337.changed.md b/changelog/4337.changed.md new file mode 100644 index 000000000..67c1ddbfe --- /dev/null +++ b/changelog/4337.changed.md @@ -0,0 +1 @@ +- If you use an STT service that emits finalized transcripts (Speechmatics, Soniox, Deepgram Flux, AssemblyAI) with `SpeechTimeoutUserTurnStopStrategy`, user turns now end as soon as `user_speech_timeout` elapses after VAD stop. Previously the strategy also waited for the STT P99 latency (`ttfs_p99_latency`) even when the transcript was already marked final. `user_speech_timeout` is still honored as a floor — STT finalization never shortens it. From 2aec2467cb93ca084f248fc527bc4a4e1e4bdd5d Mon Sep 17 00:00:00 2001 From: dhruvladia-sarvam Date: Tue, 21 Apr 2026 00:19:49 +0530 Subject: [PATCH 25/49] Deprecated InputParams fix and default model change to saaras:v3 --- src/pipecat/services/sarvam/stt.py | 44 +----------------------------- 1 file changed, 1 insertion(+), 43 deletions(-) diff --git a/src/pipecat/services/sarvam/stt.py b/src/pipecat/services/sarvam/stt.py index 4ebe703fa..7c949ac59 100644 --- a/src/pipecat/services/sarvam/stt.py +++ b/src/pipecat/services/sarvam/stt.py @@ -228,26 +228,6 @@ class SarvamSTTService(STTService): verbatim, translit, codemix. Defaults to "transcribe" for saaras:v3. vad_signals: Enable VAD signals in response. Defaults to None. high_vad_sensitivity: Enable high VAD sensitivity. Defaults to None. - positive_speech_threshold: VAD probability threshold (0.0-1.0) above which - a frame is considered speech. Only for saaras:v3. - negative_speech_threshold: VAD probability threshold (0.0-1.0) below which - a frame is considered silence. Only for saaras:v3. - min_speech_frames: Minimum consecutive speech frames to start a segment. - Only for saaras:v3. - first_turn_min_speech_frames: Minimum speech frames for the first user - turn. Only for saaras:v3. - negative_frames_count: Silence frames within the window to end a segment. - Only for saaras:v3. - negative_frames_window: Sliding window size (in frames) for counting - negative frames. Only for saaras:v3. - start_speech_volume_threshold: Volume (dB) below which audio is too quiet. - Only for saaras:v3. - interrupt_min_speech_frames: Minimum speech frames for barge-in. - Only for saaras:v3. - pre_speech_pad_frames: Audio frames to prepend before speech onset. - Only for saaras:v3. - num_initial_ignored_frames: Leading frames to skip at connection start. - Only for saaras:v3. """ language: Language | None = None @@ -255,16 +235,6 @@ class SarvamSTTService(STTService): mode: Literal["transcribe", "translate", "verbatim", "translit", "codemix"] | None = None vad_signals: bool | None = None high_vad_sensitivity: bool | None = None - positive_speech_threshold: float | None = None - negative_speech_threshold: float | None = None - min_speech_frames: int | None = None - first_turn_min_speech_frames: int | None = None - negative_frames_count: int | None = None - negative_frames_window: int | None = None - start_speech_volume_threshold: float | None = None - interrupt_min_speech_frames: int | None = None - pre_speech_pad_frames: int | None = None - num_initial_ignored_frames: int | None = None def __init__( self, @@ -311,7 +281,7 @@ class SarvamSTTService(STTService): """ # --- 1. Hardcoded defaults --- default_settings = self.Settings( - model="saarika:v2.5", + model="saaras:v3", language=None, prompt=None, vad_signals=None, @@ -343,18 +313,6 @@ class SarvamSTTService(STTService): mode = params.mode default_settings.vad_signals = params.vad_signals default_settings.high_vad_sensitivity = params.high_vad_sensitivity - default_settings.positive_speech_threshold = params.positive_speech_threshold - default_settings.negative_speech_threshold = params.negative_speech_threshold - default_settings.min_speech_frames = params.min_speech_frames - default_settings.first_turn_min_speech_frames = params.first_turn_min_speech_frames - default_settings.negative_frames_count = params.negative_frames_count - default_settings.negative_frames_window = params.negative_frames_window - default_settings.start_speech_volume_threshold = ( - params.start_speech_volume_threshold - ) - default_settings.interrupt_min_speech_frames = params.interrupt_min_speech_frames - default_settings.pre_speech_pad_frames = params.pre_speech_pad_frames - default_settings.num_initial_ignored_frames = params.num_initial_ignored_frames # --- 4. Settings delta (canonical API, always wins) --- if settings is not None: From 34fb303c44aaf553ae14ed9d082aec8233d3cef2 Mon Sep 17 00:00:00 2001 From: dhruvladia-sarvam Date: Tue, 21 Apr 2026 00:29:38 +0530 Subject: [PATCH 26/49] changelog descriptions --- changelog/4334.added.md | 1 + changelog/4334.changed.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog/4334.added.md create mode 100644 changelog/4334.changed.md diff --git a/changelog/4334.added.md b/changelog/4334.added.md new file mode 100644 index 000000000..e3dc45543 --- /dev/null +++ b/changelog/4334.added.md @@ -0,0 +1 @@ +- Added fine-grained server-side VAD tuning options to `SarvamSTTService.Settings` for the `saaras:v3` model, including speech thresholds, frame-count controls, pre-speech padding, interruption sensitivity, and initial-frame skipping. \ No newline at end of file diff --git a/changelog/4334.changed.md b/changelog/4334.changed.md new file mode 100644 index 000000000..ff8d75147 --- /dev/null +++ b/changelog/4334.changed.md @@ -0,0 +1 @@ +- `SarvamSTTService` now uses `saaras:v3` as its default model instead of `saarika:v2.5`. Applications that relied on the previous default should set `settings=SarvamSTTService.Settings(model="saarika:v2.5")` explicitly. \ No newline at end of file From c1b3a9f4b508ec9cd5d5739762e3e2b7df65d93e Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 20 Apr 2026 20:40:54 -0400 Subject: [PATCH 27/49] Add pipecat label to update-docs CI workflow --- .github/workflows/update-docs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/update-docs.yml b/.github/workflows/update-docs.yml index d26862766..bb0227233 100644 --- a/.github/workflows/update-docs.yml +++ b/.github/workflows/update-docs.yml @@ -114,6 +114,7 @@ jobs: GH_TOKEN=$DOCS_SYNC_TOKEN gh pr create \ --repo pipecat-ai/docs \ --label auto-docs \ + --label pipecat \ --title "docs: update for pipecat PR #${{ steps.pr.outputs.number }}" \ --body "$(cat <<'BODY' Automated documentation update for [pipecat PR #${{ steps.pr.outputs.number }}](https://github.com/pipecat-ai/pipecat/pull/${{ steps.pr.outputs.number }}). From a0f79b47002d946410e552f5cfcbec726aed3975 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 21 Apr 2026 09:09:19 -0400 Subject: [PATCH 28/49] Use ElevenLabs normalized_alignment so word timestamps match spoken audio --- src/pipecat/services/elevenlabs/tts.py | 77 ++++++++++++++++++++------ uv.lock | 10 ++-- 2 files changed, 66 insertions(+), 21 deletions(-) diff --git a/src/pipecat/services/elevenlabs/tts.py b/src/pipecat/services/elevenlabs/tts.py index 02e6383ff..f72864ef4 100644 --- a/src/pipecat/services/elevenlabs/tts.py +++ b/src/pipecat/services/elevenlabs/tts.py @@ -245,6 +245,35 @@ class ElevenLabsHttpTTSSettings(TTSSettings): ) +def _strip_leading_space( + alignment: Mapping[str, Any], keys: tuple[str, str, str] +) -> Mapping[str, Any]: + """Return alignment with a prepended space char removed, if present. + + Normalized alignment chunks from ElevenLabs begin with a leading space that + marks the prosody/chunk boundary. Left in place, it would prematurely + terminate a partial word carried over from the previous chunk. Stripping it + is lossless for timing: the dropped space's duration is still reflected in + the next char's `charStartTimesMs`, and the chunk's last-element values + (used to advance cumulative time) are untouched. + + Args: + alignment: Alignment dict from the API. + keys: Tuple of (chars_key, start_times_key, durations_or_end_times_key) + naming the three parallel arrays — these differ between the + WebSocket and HTTP response schemas. + """ + chars_key, starts_key, tail_key = keys + chars = alignment.get(chars_key) or [] + if chars and chars[0] == " ": + return { + chars_key: chars[1:], + starts_key: alignment.get(starts_key, [])[1:], + tail_key: alignment.get(tail_key, [])[1:], + } + return alignment + + def calculate_word_times( alignment_info: Mapping[str, Any], cumulative_time: float, @@ -790,8 +819,15 @@ class ElevenLabsTTSService(WebsocketTTSService): frame = TTSAudioRawFrame(audio, self.sample_rate, 1, context_id=received_ctx_id) await self.append_to_audio_context(received_ctx_id, frame) - if msg.get("alignment"): - alignment = msg["alignment"] + if msg.get("normalizedAlignment"): + # Use normalizedAlignment (what was actually spoken) rather than + # alignment (the input text), so word timestamps stay accurate + # when a pronunciation dictionary or text normalization rewrites + # the input. + alignment = _strip_leading_space( + msg["normalizedAlignment"], + ("chars", "charStartTimesMs", "charDurationsMs"), + ) word_times, self._partial_word, self._partial_word_start_time = ( calculate_word_times( alignment, @@ -1296,21 +1332,30 @@ class ElevenLabsHttpTTSService(TTSService): audio, self.sample_rate, 1, context_id=context_id ) - # Process alignment if present - if data and "alignment" in data: - alignment = data["alignment"] - if alignment: # Ensure alignment is not None - # Get end time of the last character in this chunk - char_end_times = alignment.get("character_end_times_seconds", []) - if char_end_times: - chunk_end_time = char_end_times[-1] - # Update to the longest end time seen so far - utterance_duration = max(utterance_duration, chunk_end_time) + # Process alignment if present. Use normalized_alignment + # (what was actually spoken) so word timestamps stay + # accurate when a pronunciation dictionary or text + # normalization rewrites the input. + if data and data.get("normalized_alignment"): + alignment = _strip_leading_space( + data["normalized_alignment"], + ( + "characters", + "character_start_times_seconds", + "character_end_times_seconds", + ), + ) + # Get end time of the last character in this chunk + char_end_times = alignment.get("character_end_times_seconds", []) + if char_end_times: + chunk_end_time = char_end_times[-1] + # Update to the longest end time seen so far + utterance_duration = max(utterance_duration, chunk_end_time) - # Calculate word timestamps - word_times = self.calculate_word_times(alignment) - if word_times: - await self.add_word_timestamps(word_times, context_id) + # Calculate word timestamps + word_times = self.calculate_word_times(alignment) + if word_times: + await self.add_word_timestamps(word_times, context_id) except json.JSONDecodeError as e: logger.warning(f"Failed to parse JSON from stream: {e}") continue diff --git a/uv.lock b/uv.lock index 49ae58ec4..16e9e5e66 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.14'", @@ -4522,7 +4522,7 @@ requires-dist = [ { name = "requests", marker = "extra == 'kokoro'", specifier = ">=2.32.5,<3" }, { name = "requests", marker = "extra == 'piper'", specifier = ">=2.32.5,<3" }, { name = "resampy", specifier = "~=0.4.3" }, - { name = "sarvamai", marker = "extra == 'sarvam'", specifier = "==0.1.26" }, + { name = "sarvamai", marker = "extra == 'sarvam'", specifier = "==0.1.28" }, { name = "sentry-sdk", marker = "extra == 'sentry'", specifier = ">=2.28.0,<3" }, { name = "simli-ai", marker = "extra == 'simli'", specifier = "~=2.0.1" }, { name = "soundfile", marker = "extra == 'soundfile'", specifier = "~=0.13.1" }, @@ -5891,7 +5891,7 @@ wheels = [ [[package]] name = "sarvamai" -version = "0.1.26" +version = "0.1.28" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, @@ -5900,9 +5900,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d7/31/13f65e8533b667514e1cfe838d12a14494cbc5943fd8f0c101305127459b/sarvamai-0.1.26.tar.gz", hash = "sha256:d51a213c27feb33d65f5b71e4882dcdb873dc5e0d720390b7ba18d1bdeec2471", size = 113050, upload-time = "2026-03-06T16:40:36.647Z" } +sdist = { url = "https://files.pythonhosted.org/packages/23/44/57a7a37be64953bb0bec9f674e92a9f8fb7070ce6aeb44c6f22720458b40/sarvamai-0.1.28.tar.gz", hash = "sha256:bc52f0c849e429d1a4493e49b1b4b65bf06965f3936d4eb6ee3da3130452e7ea", size = 139022, upload-time = "2026-04-20T08:05:12.185Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/76/c9/c03a807ace9cafbfe26418be995e4959142a55313c9f26564586e111f31d/sarvamai-0.1.26-py3-none-any.whl", hash = "sha256:39e79ba0932f4501a2aa28f84fd2de64d34fc9a7af2b0d4ead1efa617517b3bd", size = 229057, upload-time = "2026-03-06T16:40:35.584Z" }, + { url = "https://files.pythonhosted.org/packages/84/90/95cd195a3a2ae9a9973c6a05705207e8dd97b5aea90339bc24343cb54850/sarvamai-0.1.28-py3-none-any.whl", hash = "sha256:52e71c0a0f521552d4d948ec452699b44554a56e64ee354b14da2efc9d9046b3", size = 269335, upload-time = "2026-04-20T08:05:10.712Z" }, ] [[package]] From a07bee2318ff53763fef2d7c7cd9043dff491a6c Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 21 Apr 2026 09:12:15 -0400 Subject: [PATCH 29/49] Add changelog for #4344 --- changelog/4344.fixed.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/4344.fixed.md diff --git a/changelog/4344.fixed.md b/changelog/4344.fixed.md new file mode 100644 index 000000000..4e786f65c --- /dev/null +++ b/changelog/4344.fixed.md @@ -0,0 +1 @@ +- Fixed `ElevenLabsTTSService` and `ElevenLabsHttpTTSService` emitting word timestamps and `TTSTextFrame` content that matched the input text instead of the spoken audio when a pronunciation dictionary (`pronunciation_dictionary_locators`) or text normalization rewrote the input. Both services now consume ElevenLabs' normalized alignment, so downstream consumers (captions, transcripts, context aggregation) reflect what the listener actually hears. From 81571beb1b75d644040601fb901c477b3d352091 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 21 Apr 2026 10:51:59 -0400 Subject: [PATCH 30/49] Use ExternalUserTurnStrategies, as expected, in a Deepgram Flux example --- examples/update-settings/stt/stt-deepgram-flux.py | 6 +++++- uv.lock | 10 +++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/examples/update-settings/stt/stt-deepgram-flux.py b/examples/update-settings/stt/stt-deepgram-flux.py index fc3a90f31..5db99a327 100644 --- a/examples/update-settings/stt/stt-deepgram-flux.py +++ b/examples/update-settings/stt/stt-deepgram-flux.py @@ -29,6 +29,7 @@ from pipecat.transcriptions.language import Language from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams +from pipecat.turns.user_turn_strategies import ExternalUserTurnStrategies load_dotenv(override=True) @@ -79,7 +80,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): context = LLMContext() user_aggregator, assistant_aggregator = LLMContextAggregatorPair( context, - user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + user_params=LLMUserAggregatorParams( + user_turn_strategies=ExternalUserTurnStrategies(), + vad_analyzer=SileroVADAnalyzer(), + ), ) pipeline = Pipeline( diff --git a/uv.lock b/uv.lock index 49ae58ec4..16e9e5e66 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.14'", @@ -4522,7 +4522,7 @@ requires-dist = [ { name = "requests", marker = "extra == 'kokoro'", specifier = ">=2.32.5,<3" }, { name = "requests", marker = "extra == 'piper'", specifier = ">=2.32.5,<3" }, { name = "resampy", specifier = "~=0.4.3" }, - { name = "sarvamai", marker = "extra == 'sarvam'", specifier = "==0.1.26" }, + { name = "sarvamai", marker = "extra == 'sarvam'", specifier = "==0.1.28" }, { name = "sentry-sdk", marker = "extra == 'sentry'", specifier = ">=2.28.0,<3" }, { name = "simli-ai", marker = "extra == 'simli'", specifier = "~=2.0.1" }, { name = "soundfile", marker = "extra == 'soundfile'", specifier = "~=0.13.1" }, @@ -5891,7 +5891,7 @@ wheels = [ [[package]] name = "sarvamai" -version = "0.1.26" +version = "0.1.28" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, @@ -5900,9 +5900,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d7/31/13f65e8533b667514e1cfe838d12a14494cbc5943fd8f0c101305127459b/sarvamai-0.1.26.tar.gz", hash = "sha256:d51a213c27feb33d65f5b71e4882dcdb873dc5e0d720390b7ba18d1bdeec2471", size = 113050, upload-time = "2026-03-06T16:40:36.647Z" } +sdist = { url = "https://files.pythonhosted.org/packages/23/44/57a7a37be64953bb0bec9f674e92a9f8fb7070ce6aeb44c6f22720458b40/sarvamai-0.1.28.tar.gz", hash = "sha256:bc52f0c849e429d1a4493e49b1b4b65bf06965f3936d4eb6ee3da3130452e7ea", size = 139022, upload-time = "2026-04-20T08:05:12.185Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/76/c9/c03a807ace9cafbfe26418be995e4959142a55313c9f26564586e111f31d/sarvamai-0.1.26-py3-none-any.whl", hash = "sha256:39e79ba0932f4501a2aa28f84fd2de64d34fc9a7af2b0d4ead1efa617517b3bd", size = 229057, upload-time = "2026-03-06T16:40:35.584Z" }, + { url = "https://files.pythonhosted.org/packages/84/90/95cd195a3a2ae9a9973c6a05705207e8dd97b5aea90339bc24343cb54850/sarvamai-0.1.28-py3-none-any.whl", hash = "sha256:52e71c0a0f521552d4d948ec452699b44554a56e64ee354b14da2efc9d9046b3", size = 269335, upload-time = "2026-04-20T08:05:10.712Z" }, ] [[package]] From c091232f2ff5173b57909c7a7462919575c2871e Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 20 Apr 2026 19:02:50 -0400 Subject: [PATCH 31/49] Add xAI streaming STT service New `XAISTTService` wraps xAI's real-time speech-to-text WebSocket (`wss://api.x.ai/v1/stt`). It extends `WebsocketSTTService`, authenticates with the `XAI_API_KEY` as a Bearer token on the WS handshake, and streams raw audio (PCM/mu-law/A-law) with configurable interim results, endpointing, language, multichannel, and diarization settings. - `src/pipecat/services/xai/stt.py`: new service, settings dataclass, and `language_to_xai_stt_language` helper. - `src/pipecat/services/stt_latency.py`: `XAI_TTFS_P99` default. - `pyproject.toml` / `uv.lock`: `xai` extra now pulls in `websockets-base`. - `README.md`: link to xAI STT in the services table. - `examples/voice/voice-xai.py`: swap DeepgramSTTService for XAISTTService so the xAI voice example is fully xAI. - `examples/transcription/transcription-xai.py`: new transcription-only example using the new service. --- README.md | 2 +- examples/transcription/transcription-xai.py | 89 +++++ examples/voice/voice-xai.py | 4 +- pyproject.toml | 2 +- src/pipecat/services/stt_latency.py | 1 + src/pipecat/services/xai/stt.py | 412 ++++++++++++++++++++ uv.lock | 4 + 7 files changed, 510 insertions(+), 4 deletions(-) create mode 100644 examples/transcription/transcription-xai.py create mode 100644 src/pipecat/services/xai/stt.py diff --git a/README.md b/README.md index 15be8d3eb..5ea9fc4b9 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ Catch new features, interviews, and how-tos on our [Pipecat TV](https://www.yout | Category | Services | | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Gradium](https://docs.pipecat.ai/server/services/stt/gradium), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [Mistral](https://docs.pipecat.ai/server/services/stt/mistral), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Gradium](https://docs.pipecat.ai/server/services/stt/gradium), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [Mistral](https://docs.pipecat.ai/server/services/stt/mistral), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper), [xAI](https://docs.pipecat.ai/server/services/stt/xai) | | LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [Nebius](https://docs.pipecat.ai/server/services/llm/nebius), [Novita](https://docs.pipecat.ai/server/services/llm/novita), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nvidia), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/llm/sarvam), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | | Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Camb AI](https://docs.pipecat.ai/server/services/tts/camb), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Gradium](https://docs.pipecat.ai/server/services/tts/gradium), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [Kokoro](https://docs.pipecat.ai/server/services/tts/kokoro), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Mistral](https://docs.pipecat.ai/server/services/tts/mistral), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [Resemble](https://docs.pipecat.ai/server/services/tts/resemble), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [Smallest](https://docs.pipecat.ai/server/services/tts/smallest), [Speechmatics](https://docs.pipecat.ai/server/services/tts/speechmatics), [xAI](https://docs.pipecat.ai/server/services/tts/xai), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | | Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [Grok Voice Agent](https://docs.pipecat.ai/server/services/s2s/grok), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai), [Ultravox](https://docs.pipecat.ai/server/services/s2s/ultravox), | diff --git a/examples/transcription/transcription-xai.py b/examples/transcription/transcription-xai.py new file mode 100644 index 000000000..f94dc045c --- /dev/null +++ b/examples/transcription/transcription-xai.py @@ -0,0 +1,89 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import Frame, TranscriptionFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineTask +from pipecat.processors.audio.vad_processor import VADProcessor +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.xai.stt import XAISTTService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + + +class TranscriptionLogger(FrameProcessor): + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, TranscriptionFrame): + print(f"Transcription: {frame.text}") + + # Push all frames through + await self.push_frame(frame, direction) + + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = XAISTTService( + api_key=os.getenv("XAI_API_KEY"), + ) + + tl = TranscriptionLogger() + vad_processor = VADProcessor(vad_analyzer=SileroVADAnalyzer()) + + pipeline = Pipeline([transport.input(), vad_processor, stt, tl]) + + task = PipelineTask( + pipeline, + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/voice/voice-xai.py b/examples/voice/voice-xai.py index 6d0d54e79..e7fa04350 100644 --- a/examples/voice/voice-xai.py +++ b/examples/voice/voice-xai.py @@ -22,8 +22,8 @@ from pipecat.processors.aggregators.llm_response_universal import ( ) from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.xai.llm import GrokLLMService +from pipecat.services.xai.stt import XAISTTService from pipecat.services.xai.tts import XAIHttpTTSService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams @@ -53,7 +53,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = XAISTTService(api_key=os.getenv("XAI_API_KEY")) tts = XAIHttpTTSService( api_key=os.getenv("XAI_API_KEY"), diff --git a/pyproject.toml b/pyproject.toml index b1ff2cbd2..205cf14c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -123,7 +123,7 @@ webrtc = [ "aiortc>=1.14.0,<2", "opencv-python>=4.11.0.86,<5" ] websocket = [ "pipecat-ai[websockets-base]", "fastapi>=0.115.6,<1" ] websockets-base = [ "websockets>=13.1,<16.0" ] whisper = [ "faster-whisper~=1.2.1" ] -xai = [] +xai = [ "pipecat-ai[websockets-base]" ] [dependency-groups] dev = [ diff --git a/src/pipecat/services/stt_latency.py b/src/pipecat/services/stt_latency.py index 5ffd798de..7d285e353 100644 --- a/src/pipecat/services/stt_latency.py +++ b/src/pipecat/services/stt_latency.py @@ -57,3 +57,4 @@ WHISPER_TTFS_P99: float = DEFAULT_TTFS_P99 # No benchmark available yet; using conservative default MISTRAL_TTFS_P99: float = DEFAULT_TTFS_P99 SMALLEST_TTFS_P99: float = DEFAULT_TTFS_P99 +XAI_TTFS_P99: float = DEFAULT_TTFS_P99 diff --git a/src/pipecat/services/xai/stt.py b/src/pipecat/services/xai/stt.py new file mode 100644 index 000000000..9d5a5e176 --- /dev/null +++ b/src/pipecat/services/xai/stt.py @@ -0,0 +1,412 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""xAI speech-to-text service implementation. + +This module provides integration with xAI's real-time speech-to-text WebSocket +API documented at https://docs.x.ai/developers/rest-api-reference/inference/voice. +""" + +import asyncio +import json +from collections.abc import AsyncGenerator +from dataclasses import dataclass, field +from typing import Any +from urllib.parse import urlencode + +from loguru import logger + +from pipecat import version as pipecat_version +from pipecat.frames.frames import ( + CancelFrame, + EndFrame, + Frame, + InterimTranscriptionFrame, + StartFrame, + TranscriptionFrame, +) +from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven +from pipecat.services.stt_latency import XAI_TTFS_P99 +from pipecat.services.stt_service import WebsocketSTTService +from pipecat.transcriptions.language import Language, resolve_language +from pipecat.utils.time import time_now_iso8601 +from pipecat.utils.tracing.service_decorators import traced_stt + +try: + from websockets.asyncio.client import connect as websocket_connect + from websockets.protocol import State +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error('In order to use xAI STT, you need to `pip install "pipecat-ai[xai]"`.') + raise Exception(f"Missing module: {e}") + + +def language_to_xai_stt_language(language: Language) -> str | None: + """Convert a Language enum to the xAI STT language code. + + xAI STT accepts two-letter language codes (e.g. ``en``, ``fr``, ``de``, + ``ja``). When set, the server applies Inverse Text Normalization. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding xAI STT language code, or None if not supported. + """ + LANGUAGE_MAP = { + Language.AR: "ar", + Language.BN: "bn", + Language.DE: "de", + Language.EN: "en", + Language.ES: "es", + Language.FR: "fr", + Language.HI: "hi", + Language.ID: "id", + Language.IT: "it", + Language.JA: "ja", + Language.KO: "ko", + Language.PT: "pt", + Language.RU: "ru", + Language.TR: "tr", + Language.VI: "vi", + Language.ZH: "zh", + } + + return resolve_language(language, LANGUAGE_MAP, use_base_code=True) + + +@dataclass +class XAISTTSettings(STTSettings): + """Settings for XAISTTService. + + Parameters: + interim_results: When True, partial transcripts are emitted + approximately every 500ms. + endpointing: Silence duration in milliseconds that triggers a + speech-final event. Range 0-5000. Server default is 10ms. + multichannel: When True, transcribes each interleaved channel + independently. Requires ``channels`` >= 2. + channels: Number of interleaved channels (2-8). Required when + ``multichannel`` is True. + diarize: When True, the server attaches a ``speaker`` field to each + word identifying the detected speaker. + """ + + interim_results: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + endpointing: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + multichannel: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + channels: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + diarize: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + +class XAISTTService(WebsocketSTTService): + """xAI real-time speech-to-text service. + + Streams audio to xAI's WebSocket STT endpoint and emits interim and final + transcription frames. The ``XAI_API_KEY`` is passed directly as a Bearer + token on the WebSocket handshake. + + The connection is persistent: audio is streamed continuously and the + server emits ``transcript.partial`` events with ``is_final`` and + ``speech_final`` flags to mark utterance boundaries. If the connection + drops mid-session, the base class reconnects automatically. + """ + + Settings = XAISTTSettings + _settings: Settings + + def __init__( + self, + *, + api_key: str, + ws_url: str = "wss://api.x.ai/v1/stt", + sample_rate: int = 16000, + encoding: str = "pcm", + settings: Settings | None = None, + ttfs_p99_latency: float | None = XAI_TTFS_P99, + **kwargs, + ): + """Initialize the xAI STT service. + + Args: + api_key: xAI API key (used as Bearer for the WebSocket handshake). + ws_url: WebSocket endpoint URL. Defaults to ``wss://api.x.ai/v1/stt``. + sample_rate: Audio sample rate in Hz. Supported values: 8000, + 16000, 22050, 24000, 44100, 48000. Defaults to 16000. + encoding: Audio encoding. One of ``"pcm"`` (signed 16-bit LE), + ``"mulaw"``, or ``"alaw"``. Defaults to ``"pcm"``. + settings: Runtime-updatable settings overriding defaults. + ttfs_p99_latency: P99 latency from speech end to final transcript + in seconds. See https://github.com/pipecat-ai/stt-benchmark. + **kwargs: Additional arguments passed to WebsocketSTTService. + """ + default_settings = self.Settings( + model=None, + language=Language.EN, + interim_results=True, + endpointing=None, + multichannel=None, + channels=None, + diarize=None, + ) + if settings is not None: + default_settings.apply_update(settings) + + super().__init__( + sample_rate=sample_rate, + settings=default_settings, + ttfs_p99_latency=ttfs_p99_latency, + **kwargs, + ) + + self._api_key = api_key + self._ws_url = ws_url + self._encoding = encoding + + self._receive_task: asyncio.Task | None = None + self._session_ready = asyncio.Event() + + def can_generate_metrics(self) -> bool: + """Check if the service can generate metrics. + + Returns: + True if metrics generation is supported. + """ + return True + + def language_to_service_language(self, language: Language) -> str | None: + """Convert a Language enum to the xAI STT language code.""" + return language_to_xai_stt_language(language) + + async def _update_settings(self, delta: Settings) -> dict[str, Any]: + """Apply a settings delta and reconnect to apply changes. + + xAI STT configures the session via WebSocket query parameters, so any + change requires a fresh connection. + """ + changed = await super()._update_settings(delta) + if not changed: + return changed + await self._disconnect() + await self._connect() + return changed + + async def start(self, frame: StartFrame): + """Start the speech-to-text service.""" + await super().start(frame) + await self._connect() + + async def stop(self, frame: EndFrame): + """Stop the speech-to-text service.""" + await super().stop(frame) + await self._disconnect() + + async def cancel(self, frame: CancelFrame): + """Cancel the speech-to-text service.""" + await super().cancel(frame) + await self._disconnect() + + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Forward raw audio bytes to the xAI STT WebSocket. + + Transcription frames are pushed from the receive task, not yielded + from this coroutine. + """ + if self._websocket and self._websocket.state is State.OPEN and self._session_ready.is_set(): + try: + await self._websocket.send(audio) + except Exception as e: + await self.push_error(error_msg=f"xAI STT send failed: {e}", exception=e) + yield None + + def _build_ws_url(self) -> str: + """Build the WebSocket URL with session query parameters.""" + s = self._settings + params: dict[str, Any] = { + "sample_rate": self.sample_rate, + "encoding": self._encoding, + } + + if s.language is not None: + params["language"] = s.language + + optional_fields = { + "interim_results": s.interim_results, + "endpointing": s.endpointing, + "multichannel": s.multichannel, + "channels": s.channels, + "diarize": s.diarize, + } + for key, val in optional_fields.items(): + if val is None: + continue + if isinstance(val, bool): + params[key] = str(val).lower() + else: + params[key] = val + + return f"{self._ws_url}?{urlencode(params)}" + + async def _connect(self): + """Establish the WebSocket connection and start the receive task.""" + await super()._connect() + await self._connect_websocket() + if self._websocket and not self._receive_task: + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) + + async def _disconnect(self): + """Tear down the WebSocket connection and cancel the receive task.""" + await super()._disconnect() + try: + if self._websocket and self._websocket.state is State.OPEN: + await self._websocket.send(json.dumps({"type": "audio.done"})) + except Exception as e: + logger.debug(f"{self} error sending audio.done during disconnect: {e}") + + if self._receive_task: + await self.cancel_task(self._receive_task) + self._receive_task = None + + await self._disconnect_websocket() + + async def _connect_websocket(self): + """Open a WebSocket connection to the xAI STT endpoint.""" + try: + if self._websocket and self._websocket.state is State.OPEN: + return + + logger.debug("Connecting to xAI STT WebSocket") + self._session_ready.clear() + + ws_url = self._build_ws_url() + headers = { + "Authorization": f"Bearer {self._api_key}", + "User-Agent": f"xAI/1.0 (integration=Pipecat/{pipecat_version()})", + } + self._websocket = await websocket_connect(ws_url, additional_headers=headers) + await self._call_event_handler("on_connected") + logger.debug(f"{self} connected to xAI STT WebSocket") + except Exception as e: + await self.push_error(error_msg=f"Unable to connect to xAI STT: {e}", exception=e) + raise + + async def _disconnect_websocket(self): + """Close the WebSocket connection.""" + try: + if self._websocket: + logger.debug("Disconnecting from xAI STT WebSocket") + await self._websocket.close() + except Exception as e: + await self.push_error(error_msg=f"Error closing xAI STT websocket: {e}", exception=e) + finally: + self._websocket = None + self._session_ready.clear() + await self._call_event_handler("on_disconnected") + + async def _receive_messages(self): + """Receive and dispatch xAI STT WebSocket messages.""" + if not self._websocket: + raise Exception("Websocket not connected") + async for message in self._websocket: + try: + data = json.loads(message) + except json.JSONDecodeError: + logger.warning(f"{self} received non-JSON message: {message}") + continue + await self._handle_message(data) + + async def _handle_message(self, message: dict[str, Any]): + """Branch on xAI STT event type.""" + msg_type = message.get("type") + print(f"xAI STT message: {message}") + + if msg_type == "transcript.created": + self._session_ready.set() + logger.debug(f"{self} xAI STT session ready") + elif msg_type == "transcript.partial": + await self._handle_transcript(message) + elif msg_type == "transcript.done": + if message.get("text"): + await self._push_final_transcript(message, speech_final=True) + elif msg_type == "error": + await self.push_error( + error_msg=f"xAI STT error: {message.get('message', message)}", + exception=Exception(message), + ) + else: + logger.debug(f"{self} unhandled xAI STT message: {message}") + + async def _handle_transcript(self, message: dict[str, Any]): + text = message.get("text", "") + if not text: + return + + is_final = bool(message.get("is_final")) + speech_final = bool(message.get("speech_final")) + language = self._language_for_frame() + + if is_final: + await self._push_final_transcript( + message, speech_final=speech_final, language=language, text=text + ) + else: + await self.push_frame( + InterimTranscriptionFrame( + text, + self._user_id, + time_now_iso8601(), + language, + result=message, + ) + ) + + async def _push_final_transcript( + self, + message: dict[str, Any], + *, + speech_final: bool, + language: Language | None = None, + text: str | None = None, + ): + text = text if text is not None else message.get("text", "") + if not text: + return + language = language if language is not None else self._language_for_frame() + + await self.push_frame( + TranscriptionFrame( + text, + self._user_id, + time_now_iso8601(), + language, + result=message, + finalized=speech_final, + ) + ) + await self._trace_transcription(text, True, language) + if speech_final: + await self.stop_processing_metrics() + + def _language_for_frame(self) -> Language: + """Return a Language enum suitable for transcription frames. + + Settings stores the service-specific string (e.g. ``"en"``); frames + carry the enum value. + """ + lang = self._settings.language + if isinstance(lang, Language): + return lang + if isinstance(lang, str): + try: + return Language(lang) + except ValueError: + return Language.EN + return Language.EN + + @traced_stt + async def _trace_transcription(self, transcript: str, is_final: bool, language: Language): + """Record transcription event for tracing.""" + pass diff --git a/uv.lock b/uv.lock index 16e9e5e66..113fb056c 100644 --- a/uv.lock +++ b/uv.lock @@ -4403,6 +4403,9 @@ websockets-base = [ whisper = [ { name = "faster-whisper" }, ] +xai = [ + { name = "websockets" }, +] [package.dev-dependencies] dev = [ @@ -4507,6 +4510,7 @@ requires-dist = [ { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'soniox'" }, { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'ultravox'" }, { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'websocket'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'xai'" }, { name = "pipecat-ai-small-webrtc-prebuilt", marker = "extra == 'runner'", specifier = ">=2.4.0" }, { name = "piper-tts", marker = "extra == 'piper'", specifier = ">=1.3.0,<2" }, { name = "protobuf", specifier = ">=6.31.1,<7" }, From b838bd906bb38b262cccf8ec7074caf69abd0946 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 20 Apr 2026 19:03:34 -0400 Subject: [PATCH 32/49] Add changelog for #4340 --- changelog/4340.added.md | 1 + examples/transcription/transcription-xai.py | 5 +---- src/pipecat/services/xai/stt.py | 1 - 3 files changed, 2 insertions(+), 5 deletions(-) create mode 100644 changelog/4340.added.md diff --git a/changelog/4340.added.md b/changelog/4340.added.md new file mode 100644 index 000000000..5abcc0f0e --- /dev/null +++ b/changelog/4340.added.md @@ -0,0 +1 @@ +- Added `XAISTTService` for real-time speech-to-text using xAI's voice STT WebSocket API (`wss://api.x.ai/v1/stt`). Streams raw audio (PCM, µ-law, or A-law) and emits interim and final transcription frames driven by the server's `is_final` / `speech_final` flags. Settings expose `interim_results`, `endpointing`, `language`, `multichannel`, `channels`, and `diarize`. Requires the `xai` optional extra (`pip install "pipecat-ai[xai]"`). diff --git a/examples/transcription/transcription-xai.py b/examples/transcription/transcription-xai.py index f94dc045c..0cc667f9a 100644 --- a/examples/transcription/transcription-xai.py +++ b/examples/transcription/transcription-xai.py @@ -9,12 +9,10 @@ import os from dotenv import load_dotenv from loguru import logger -from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import Frame, TranscriptionFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask -from pipecat.processors.audio.vad_processor import VADProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -58,9 +56,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tl = TranscriptionLogger() - vad_processor = VADProcessor(vad_analyzer=SileroVADAnalyzer()) - pipeline = Pipeline([transport.input(), vad_processor, stt, tl]) + pipeline = Pipeline([transport.input(), stt, tl]) task = PipelineTask( pipeline, diff --git a/src/pipecat/services/xai/stt.py b/src/pipecat/services/xai/stt.py index 9d5a5e176..df8406fc2 100644 --- a/src/pipecat/services/xai/stt.py +++ b/src/pipecat/services/xai/stt.py @@ -321,7 +321,6 @@ class XAISTTService(WebsocketSTTService): async def _handle_message(self, message: dict[str, Any]): """Branch on xAI STT event type.""" msg_type = message.get("type") - print(f"xAI STT message: {message}") if msg_type == "transcript.created": self._session_ready.set() From 29d604f608a3395a4cf1428ce18d0407274f893e Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 21 Apr 2026 14:48:55 -0400 Subject: [PATCH 33/49] Fix UnboundLocalError in Deepgram STT connection handler If the WebSocket handshake is cancelled or fails before `keepalive_task` is assigned (e.g. an STTUpdateSettingsFrame triggers a reconnect during initial connect), the `finally` block tried to cancel an unbound local. Initialize `keepalive_task = None` before the try and guard the cancel. --- src/pipecat/services/deepgram/stt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/deepgram/stt.py b/src/pipecat/services/deepgram/stt.py index 66b1d70da..c9ebd49c8 100644 --- a/src/pipecat/services/deepgram/stt.py +++ b/src/pipecat/services/deepgram/stt.py @@ -621,6 +621,7 @@ class DeepgramSTTService(STTService): """ while True: connect_kwargs = self._build_connect_kwargs() + keepalive_task = None try: async with self._client.listen.v1.connect(**connect_kwargs) as connection: self._connection = connection @@ -639,7 +640,8 @@ class DeepgramSTTService(STTService): finally: self._connection_ready.clear() self._connection = None - await self.cancel_task(keepalive_task) + if keepalive_task: + await self.cancel_task(keepalive_task) async def _keepalive_handler(self): """Periodically send KeepAlive frames to prevent server-side timeout. From 648094da261569564619624a1179e6ddc59a8115 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 21 Apr 2026 14:51:30 -0400 Subject: [PATCH 34/49] Add changelog for #4347 --- changelog/4347.fixed.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/4347.fixed.md diff --git a/changelog/4347.fixed.md b/changelog/4347.fixed.md new file mode 100644 index 000000000..e5c8206e5 --- /dev/null +++ b/changelog/4347.fixed.md @@ -0,0 +1 @@ +- Fixed a crash in `DeepgramSTTService` when an `STTUpdateSettingsFrame` arrived before the WebSocket handshake completed (for example, when pushing an update upstream on `StartFrame`). The settings-triggered reconnect cancelled the in-flight connection task before its keepalive task was created, causing an `UnboundLocalError: cannot access local variable 'keepalive_task'` in the handler's `finally` block. From 58a17c7b1bde8deb899e10d68f0abef979850245 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 20 Apr 2026 15:50:49 -0400 Subject: [PATCH 35/49] Include examples in type checking Remove `examples/` from the `pyrightconfig.json` ignore list and fix the resulting type errors across all example files. Common fixes: - Required API keys: `os.getenv("X")` -> `os.environ["X"]` so the return type is `str` rather than `str | None`, and misconfiguration fails fast. - Narrow `LLMContextMessage` union members with `isinstance(..., dict)` before dict-style access. - `assert isinstance(params.llm, ...)` before calling service-specific methods that aren't on the base `LLMService`. - Guard optional frame fields (e.g. `LLMSearchResponseFrame.search_result`) before use. --- examples/audio/audio-bot-background-sound.py | 6 +- examples/audio/audio-recording.py | 6 +- examples/audio/audio-sound-effects.py | 6 +- .../context-summarization-dedicated-llm.py | 8 +- .../context-summarization-google.py | 6 +- .../context-summarization-manual-openai.py | 6 +- .../context-summarization-openai.py | 6 +- .../features-before-and-after-events.py | 6 +- .../features-concurrent-llm-evaluation.py | 8 +- ...res-concurrent-llm-rtvi-ignored-sources.py | 8 +- .../features-custom-frame-processor.py | 6 +- .../features-gpu-container-local-bot.py | 6 +- .../features/features-live-translation.py | 6 +- .../features-pattern-pair-voice-switching.py | 6 +- .../features/features-service-switcher.py | 12 +- .../features/features-switch-languages.py | 8 +- examples/features/features-switch-voices.py | 10 +- .../features/features-user-email-gathering.py | 6 +- .../features/features-voicemail-detection.py | 8 +- examples/features/features-wake-phrase.py | 6 +- ...function-calling-anthropic-async-stream.py | 6 +- .../function-calling-anthropic-async.py | 6 +- .../function-calling-anthropic-video.py | 6 +- .../function-calling-anthropic.py | 6 +- .../function-calling-aws-video.py | 4 +- .../function-calling-azure.py | 8 +- .../function-calling-cerebras.py | 6 +- .../function-calling-deepseek.py | 6 +- .../function-calling-direct.py | 6 +- .../function-calling-fireworks.py | 6 +- .../function-calling-google-async-stream.py | 6 +- .../function-calling-google-async.py | 6 +- .../function-calling-google-vertex.py | 27 ++-- .../function-calling-google-video.py | 6 +- .../function-calling-google.py | 6 +- .../function-calling/function-calling-grok.py | 6 +- .../function-calling/function-calling-groq.py | 6 +- .../function-calling-mistral.py | 6 +- .../function-calling-moondream-video.py | 6 +- .../function-calling-nebius.py | 6 +- .../function-calling-novita.py | 6 +- .../function-calling-nvidia.py | 6 +- .../function-calling-ollama.py | 4 +- .../function-calling-openai-async-stream.py | 6 +- .../function-calling-openai-async.py | 6 +- ...n-calling-openai-responses-async-stream.py | 6 +- ...function-calling-openai-responses-async.py | 6 +- .../function-calling-openai-responses-http.py | 6 +- ...ion-calling-openai-responses-video-http.py | 6 +- ...function-calling-openai-responses-video.py | 6 +- .../function-calling-openai-responses.py | 6 +- .../function-calling-openai-video.py | 6 +- .../function-calling-openai.py | 6 +- .../function-calling-openrouter.py | 8 +- .../function-calling-perplexity.py | 6 +- .../function-calling/function-calling-qwen.py | 6 +- .../function-calling-sambanova.py | 6 +- .../function-calling-sarvam.py | 6 +- .../function-calling-together.py | 6 +- examples/getting-started/01-say-one-thing.py | 2 +- examples/getting-started/01a-local-audio.py | 2 +- .../getting-started/02-llm-say-one-thing.py | 4 +- examples/getting-started/03-still-frame.py | 2 +- .../04-sync-speech-and-image.py | 4 +- examples/getting-started/05-speaking-state.py | 6 +- examples/getting-started/06-voice-agent.py | 6 +- .../getting-started/06a-voice-agent-local.py | 6 +- .../getting-started/07-function-calling.py | 6 +- examples/mcp/mcp-multiple-mcp.py | 6 +- examples/mcp/mcp-stdio.py | 6 +- .../mcp/mcp-streamable-http-gemini-live.py | 13 +- examples/mcp/mcp-streamable-http.py | 6 +- .../observability/observability-observer.py | 6 +- .../observability-sentry-metrics.py | 6 +- .../persistent-context-anthropic.py | 6 +- .../persistent-context-aws-nova-sonic.py | 11 +- .../persistent-context-gemini.py | 6 +- .../persistent-context-grok-realtime.py | 2 +- .../persistent-context-openai-realtime.py | 5 +- ...ersistent-context-openai-responses-http.py | 6 +- .../persistent-context-openai-responses.py | 6 +- .../persistent-context-openai.py | 6 +- examples/rag/rag-gemini-grounding-metadata.py | 6 +- examples/rag/rag-gemini.py | 8 +- examples/rag/rag-mem0.py | 6 +- examples/realtime/realtime-aws-nova-sonic.py | 6 +- examples/realtime/realtime-azure.py | 4 +- .../realtime-gemini-live-files-api.py | 2 +- .../realtime-gemini-live-function-calling.py | 2 +- .../realtime-gemini-live-google-search.py | 2 +- .../realtime-gemini-live-graceful-end.py | 2 +- ...realtime-gemini-live-grounding-metadata.py | 21 ++- .../realtime-gemini-live-local-vad.py | 2 +- ...ime-gemini-live-vertex-function-calling.py | 4 +- .../realtime/realtime-gemini-live-video.py | 2 +- examples/realtime/realtime-gemini-live.py | 2 +- examples/realtime/realtime-grok.py | 2 +- examples/realtime/realtime-inworld.py | 2 +- .../realtime/realtime-openai-live-video.py | 6 +- examples/realtime/realtime-openai-text.py | 4 +- examples/realtime/realtime-openai.py | 2 +- examples/realtime/realtime-ultravox-text.py | 2 +- examples/realtime/realtime-ultravox.py | 2 +- examples/thinking/thinking-anthropic.py | 6 +- .../thinking/thinking-functions-anthropic.py | 6 +- .../thinking/thinking-functions-google.py | 6 +- examples/thinking/thinking-google.py | 6 +- .../transcription/transcription-assemblyai.py | 2 +- examples/transcription/transcription-azure.py | 2 +- .../transcription/transcription-cartesia.py | 2 +- .../transcription/transcription-deepgram.py | 2 +- .../transcription/transcription-elevenlabs.py | 2 +- .../transcription-gladia-translation.py | 7 +- .../transcription/transcription-gladia.py | 7 +- .../transcription/transcription-google-llm.py | 6 +- .../transcription/transcription-gradium.py | 2 +- .../transcription/transcription-mistral.py | 2 +- .../transcription/transcription-openai.py | 2 +- .../transcription/transcription-soniox.py | 2 +- .../transcription-speechmatics.py | 2 +- examples/transports/transports-daily.py | 4 +- examples/transports/transports-livekit.py | 6 +- .../transports/transports-small-webrtc.py | 6 +- .../turn-management-detect-user-idle.py | 9 +- ...turn-management-filter-incomplete-turns.py | 6 +- .../turn-management-interruption-config.py | 6 +- ...turn-management-smart-turn-local-coreml.py | 8 +- .../turn-management-smart-turn-local.py | 6 +- .../turn-management-turn-tracking-observer.py | 6 +- .../turn-management-user-assistant-turns.py | 6 +- .../turn-management-user-mute-strategy.py | 6 +- examples/update-settings/llm/llm-anthropic.py | 6 +- .../update-settings/llm/llm-aws-bedrock.py | 4 +- .../update-settings/llm/llm-aws-nova-sonic.py | 6 +- .../update-settings/llm/llm-azure-realtime.py | 7 +- examples/update-settings/llm/llm-azure.py | 8 +- examples/update-settings/llm/llm-cerebras.py | 6 +- examples/update-settings/llm/llm-deepseek.py | 6 +- examples/update-settings/llm/llm-fireworks.py | 6 +- .../llm/llm-gemini-live-vertex.py | 6 +- .../update-settings/llm/llm-gemini-live.py | 2 +- .../update-settings/llm/llm-google-vertex.py | 8 +- examples/update-settings/llm/llm-google.py | 6 +- .../update-settings/llm/llm-grok-realtime.py | 5 +- examples/update-settings/llm/llm-grok.py | 6 +- examples/update-settings/llm/llm-groq.py | 6 +- examples/update-settings/llm/llm-mistral.py | 6 +- examples/update-settings/llm/llm-nvidia.py | 6 +- examples/update-settings/llm/llm-ollama.py | 4 +- .../llm/llm-openai-realtime.py | 5 +- .../llm/llm-openai-responses-http.py | 6 +- .../llm/llm-openai-responses.py | 6 +- examples/update-settings/llm/llm-openai.py | 6 +- .../update-settings/llm/llm-openrouter.py | 6 +- .../update-settings/llm/llm-perplexity.py | 18 +-- examples/update-settings/llm/llm-qwen.py | 6 +- examples/update-settings/llm/llm-sambanova.py | 6 +- examples/update-settings/llm/llm-sarvam.py | 21 ++- examples/update-settings/llm/llm-together.py | 6 +- .../llm/llm-ultravox-realtime.py | 5 +- .../update-settings/stt/stt-assemblyai.py | 6 +- .../update-settings/stt/stt-aws-transcribe.py | 4 +- examples/update-settings/stt/stt-azure.py | 6 +- examples/update-settings/stt/stt-cartesia.py | 6 +- .../update-settings/stt/stt-deepgram-flux.py | 6 +- .../stt/stt-deepgram-sagemaker.py | 8 +- examples/update-settings/stt/stt-deepgram.py | 6 +- .../stt/stt-elevenlabs-realtime.py | 6 +- .../update-settings/stt/stt-elevenlabs.py | 6 +- examples/update-settings/stt/stt-fal.py | 4 +- examples/update-settings/stt/stt-gladia.py | 6 +- examples/update-settings/stt/stt-google.py | 6 +- examples/update-settings/stt/stt-gradium.py | 6 +- examples/update-settings/stt/stt-groq.py | 6 +- .../stt/stt-nvidia-segmented.py | 6 +- examples/update-settings/stt/stt-nvidia.py | 6 +- .../stt/stt-openai-realtime.py | 6 +- examples/update-settings/stt/stt-sarvam.py | 6 +- examples/update-settings/stt/stt-soniox.py | 6 +- .../update-settings/stt/stt-speechmatics.py | 6 +- .../update-settings/stt/stt-whisper-api.py | 6 +- .../update-settings/stt/stt-whisper-mlx.py | 4 +- examples/update-settings/stt/stt-whisper.py | 4 +- .../update-settings/tts/tts-asyncai-http.py | 4 +- examples/update-settings/tts/tts-asyncai.py | 4 +- examples/update-settings/tts/tts-aws-polly.py | 4 +- .../update-settings/tts/tts-azure-http.py | 8 +- examples/update-settings/tts/tts-azure.py | 8 +- examples/update-settings/tts/tts-camb.py | 6 +- .../update-settings/tts/tts-cartesia-http.py | 6 +- examples/update-settings/tts/tts-cartesia.py | 6 +- .../update-settings/tts/tts-deepgram-http.py | 6 +- .../tts/tts-deepgram-sagemaker.py | 8 +- examples/update-settings/tts/tts-deepgram.py | 6 +- .../tts/tts-elevenlabs-http.py | 13 +- .../update-settings/tts/tts-elevenlabs.py | 20 ++- examples/update-settings/tts/tts-fish.py | 6 +- examples/update-settings/tts/tts-gemini.py | 6 +- .../update-settings/tts/tts-google-http.py | 6 +- .../update-settings/tts/tts-google-stream.py | 6 +- examples/update-settings/tts/tts-gradium.py | 6 +- examples/update-settings/tts/tts-groq.py | 6 +- examples/update-settings/tts/tts-hume.py | 4 +- .../update-settings/tts/tts-inworld-http.py | 6 +- examples/update-settings/tts/tts-inworld.py | 6 +- examples/update-settings/tts/tts-kokoro.py | 4 +- examples/update-settings/tts/tts-lmnt.py | 6 +- examples/update-settings/tts/tts-minimax.py | 8 +- .../update-settings/tts/tts-neuphonic-http.py | 6 +- examples/update-settings/tts/tts-neuphonic.py | 6 +- examples/update-settings/tts/tts-nvidia.py | 6 +- examples/update-settings/tts/tts-openai.py | 6 +- .../update-settings/tts/tts-piper-http.py | 6 +- examples/update-settings/tts/tts-piper.py | 4 +- .../update-settings/tts/tts-resembleai.py | 10 +- examples/update-settings/tts/tts-rime-http.py | 6 +- examples/update-settings/tts/tts-rime.py | 6 +- .../update-settings/tts/tts-sarvam-http.py | 6 +- examples/update-settings/tts/tts-sarvam.py | 6 +- .../update-settings/tts/tts-speechmatics.py | 6 +- examples/update-settings/tts/tts-xtts.py | 4 +- .../video-avatar-heygen-transport.py | 8 +- .../video-avatar-heygen-video-service.py | 8 +- .../video-avatar-lemonslice-transport.py | 6 +- .../video-avatar-simli-video-service.py | 8 +- .../video-avatar-tavus-transport.py | 10 +- .../video-avatar-tavus-video-service.py | 10 +- examples/video-processing/video-processing.py | 17 ++- examples/vision/vision-anthropic.py | 6 +- examples/vision/vision-aws.py | 4 +- examples/vision/vision-gemini-flash.py | 6 +- examples/vision/vision-moondream.py | 2 +- .../vision/vision-openai-responses-http.py | 6 +- examples/vision/vision-openai-responses.py | 6 +- examples/vision/vision-openai.py | 6 +- examples/voice/voice-aicoustics.py | 6 +- .../voice/voice-assemblyai-turn-detection.py | 6 +- examples/voice/voice-assemblyai.py | 6 +- examples/voice/voice-asyncai-http.py | 4 +- examples/voice/voice-asyncai.py | 4 +- examples/voice/voice-azure-http.py | 10 +- examples/voice/voice-azure.py | 10 +- examples/voice/voice-camb.py | 6 +- examples/voice/voice-cartesia-http.py | 6 +- examples/voice/voice-cartesia.py | 6 +- .../voice/voice-deepgram-flux-sagemaker.py | 8 +- examples/voice/voice-deepgram-flux.py | 6 +- examples/voice/voice-deepgram-http.py | 6 +- examples/voice/voice-deepgram-sagemaker.py | 8 +- examples/voice/voice-deepgram-vad.py | 133 ------------------ examples/voice/voice-deepgram.py | 6 +- examples/voice/voice-elevenlabs-http.py | 4 +- examples/voice/voice-elevenlabs.py | 4 +- examples/voice/voice-fal.py | 4 +- examples/voice/voice-fish.py | 6 +- examples/voice/voice-gladia-vad.py | 7 +- examples/voice/voice-gladia.py | 7 +- examples/voice/voice-google-audio-in.py | 4 +- examples/voice/voice-google-gemini-tts.py | 6 +- examples/voice/voice-google-http.py | 6 +- examples/voice/voice-google-image.py | 6 +- examples/voice/voice-google.py | 6 +- examples/voice/voice-gradium.py | 6 +- examples/voice/voice-groq.py | 6 +- examples/voice/voice-hume.py | 4 +- examples/voice/voice-inworld-http.py | 4 +- examples/voice/voice-inworld.py | 4 +- examples/voice/voice-kokoro.py | 4 +- examples/voice/voice-krisp-viva.py | 6 +- examples/voice/voice-langchain.py | 12 +- examples/voice/voice-lmnt.py | 6 +- examples/voice/voice-minimax.py | 4 +- examples/voice/voice-mistral.py | 6 +- examples/voice/voice-neuphonic-http.py | 6 +- examples/voice/voice-neuphonic.py | 6 +- examples/voice/voice-nvidia.py | 6 +- examples/voice/voice-openai-http.py | 6 +- examples/voice/voice-openai-responses-http.py | 6 +- examples/voice/voice-openai-responses.py | 6 +- examples/voice/voice-openai.py | 6 +- examples/voice/voice-piper.py | 4 +- examples/voice/voice-resemble.py | 8 +- examples/voice/voice-rime-http.py | 4 +- examples/voice/voice-rime.py | 4 +- examples/voice/voice-sarvam-http.py | 6 +- examples/voice/voice-sarvam.py | 6 +- examples/voice/voice-smallest.py | 6 +- examples/voice/voice-soniox.py | 6 +- examples/voice/voice-speechmatics-vad.py | 6 +- examples/voice/voice-speechmatics.py | 6 +- examples/voice/voice-xai.py | 6 +- examples/voice/voice-xtts.py | 4 +- pyrightconfig.json | 6 +- 293 files changed, 884 insertions(+), 1006 deletions(-) delete mode 100644 examples/voice/voice-deepgram-vad.py diff --git a/examples/audio/audio-bot-background-sound.py b/examples/audio/audio-bot-background-sound.py index 4237bb625..6417e5549 100644 --- a/examples/audio/audio-bot-background-sound.py +++ b/examples/audio/audio-bot-background-sound.py @@ -71,17 +71,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/audio/audio-recording.py b/examples/audio/audio-recording.py index 68354e0dd..9ece2b796 100644 --- a/examples/audio/audio-recording.py +++ b/examples/audio/audio-recording.py @@ -108,17 +108,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"), audio_passthrough=True) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"], audio_passthrough=True) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/audio/audio-sound-effects.py b/examples/audio/audio-sound-effects.py index 1f7fdd339..19ed85675 100644 --- a/examples/audio/audio-sound-effects.py +++ b/examples/audio/audio-sound-effects.py @@ -102,17 +102,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), diff --git a/examples/context-summarization/context-summarization-dedicated-llm.py b/examples/context-summarization/context-summarization-dedicated-llm.py index e4de35cb7..bc13e9c46 100644 --- a/examples/context-summarization/context-summarization-dedicated-llm.py +++ b/examples/context-summarization/context-summarization-dedicated-llm.py @@ -89,10 +89,10 @@ async def get_current_weather(params: FunctionCallParams): async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info("Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -109,7 +109,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Primary LLM for conversation (could be any provider) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction=system_prompt, ), @@ -117,7 +117,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Dedicated cheap/fast LLM for summarization only summarization_llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( model="gemini-2.5-flash", ), diff --git a/examples/context-summarization/context-summarization-google.py b/examples/context-summarization/context-summarization-google.py index f8cf50419..3b0daca32 100644 --- a/examples/context-summarization/context-summarization-google.py +++ b/examples/context-summarization/context-summarization-google.py @@ -77,17 +77,17 @@ async def get_current_weather(params: FunctionCallParams): async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info("Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You have access to tools to get the current weather - use them when relevant.", ), diff --git a/examples/context-summarization/context-summarization-manual-openai.py b/examples/context-summarization/context-summarization-manual-openai.py index 017831695..23f70d97c 100644 --- a/examples/context-summarization/context-summarization-manual-openai.py +++ b/examples/context-summarization/context-summarization-manual-openai.py @@ -72,10 +72,10 @@ async def summarize_conversation(params: FunctionCallParams): async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info("Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -91,7 +91,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): """ llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction=system_prompt, ), diff --git a/examples/context-summarization/context-summarization-openai.py b/examples/context-summarization/context-summarization-openai.py index 53c5b26a5..05566f32b 100644 --- a/examples/context-summarization/context-summarization-openai.py +++ b/examples/context-summarization/context-summarization-openai.py @@ -77,17 +77,17 @@ async def get_current_weather(params: FunctionCallParams): async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info("Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You have access to tools to get the current weather - use them when relevant.", ), diff --git a/examples/features/features-before-and-after-events.py b/examples/features/features-before-and-after-events.py index b9111b945..dedf3c4fa 100644 --- a/examples/features/features-before-and-after-events.py +++ b/examples/features/features-before-and-after-events.py @@ -63,17 +63,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/features/features-concurrent-llm-evaluation.py b/examples/features/features-concurrent-llm-evaluation.py index 25b9372d0..cb085a8c7 100644 --- a/examples/features/features-concurrent-llm-evaluation.py +++ b/examples/features/features-concurrent-llm-evaluation.py @@ -58,24 +58,24 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) openai_llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), ) groq_llm = GroqLLMService( - api_key=os.getenv("GROQ_API_KEY"), + api_key=os.environ["GROQ_API_KEY"], settings=GroqLLMService.Settings( system_instruction="You are a very helpful assistant. Your goal is to demonstrate your capabilities in detail in a creative and helpful way.", ), diff --git a/examples/features/features-concurrent-llm-rtvi-ignored-sources.py b/examples/features/features-concurrent-llm-rtvi-ignored-sources.py index 678c56a0b..70530b51e 100644 --- a/examples/features/features-concurrent-llm-rtvi-ignored-sources.py +++ b/examples/features/features-concurrent-llm-rtvi-ignored-sources.py @@ -63,10 +63,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info("Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -74,7 +74,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Main LLM — drives the conversation. Its RTVI events reach the client. main_llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), @@ -83,7 +83,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Evaluator LLM — silently grades the user's message in the background. # Its RTVI events will be suppressed so the client is unaware of this branch. evaluator_llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], name="EvaluatorLLM", settings=OpenAILLMService.Settings( system_instruction="You are a silent quality evaluator. When given a user message, respond with a single JSON object: {'score': <1-5>, 'reason': ''}. Do not respond conversationally.", diff --git a/examples/features/features-custom-frame-processor.py b/examples/features/features-custom-frame-processor.py index bf143c40d..ab82bae80 100644 --- a/examples/features/features-custom-frame-processor.py +++ b/examples/features/features-custom-frame-processor.py @@ -91,17 +91,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/features/features-gpu-container-local-bot.py b/examples/features/features-gpu-container-local-bot.py index c61870866..67a8dabc7 100644 --- a/examples/features/features-gpu-container-local-bot.py +++ b/examples/features/features-gpu-container-local-bot.py @@ -56,10 +56,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = DeepgramTTSService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], settings=DeepgramTTSService.Settings( voice="aura-asteria-en", ), @@ -68,7 +68,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = OpenAILLMService( # To use OpenAI - # api_key=os.getenv("OPENAI_API_KEY"), + # api_key=os.environ["OPENAI_API_KEY"], # Or, to use a local vLLM (or similar) api server settings=OpenAILLMService.Settings( model="meta-llama/Meta-Llama-3-8B-Instruct", diff --git a/examples/features/features-live-translation.py b/examples/features/features-live-translation.py index 7a4a4b112..e4e7d075f 100644 --- a/examples/features/features-live-translation.py +++ b/examples/features/features-live-translation.py @@ -55,17 +55,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="d4db5fb9-f44b-4bd1-85fa-192e0f0d75f9", # Spanish-speaking Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a live translation assistant. Your sole purpose is to translate English text into Spanish. When you receive English text from the user, immediately translate it into natural, fluent Spanish. Do not add explanations, commentary, or extra information—only provide the Spanish translation of the text you receive.", ), diff --git a/examples/features/features-pattern-pair-voice-switching.py b/examples/features/features-pattern-pair-voice-switching.py index 38926d915..1f5b83cb8 100644 --- a/examples/features/features-pattern-pair-voice-switching.py +++ b/examples/features/features-pattern-pair-voice-switching.py @@ -126,14 +126,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm_text_aggregator.on_pattern_match("voice", on_voice_tag) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) # Process LLM text through the pattern aggregator before TTS llm_text_processor = LLMTextProcessor(text_aggregator=llm_text_aggregator) # Initialize TTS with narrator voice as default tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice=VOICE_IDS["narrator"], ), @@ -190,7 +190,7 @@ Remember: Use narrator voice for EVERYTHING except the actual quoted dialogue."" # Initialize LLM llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction=system_prompt, ), diff --git a/examples/features/features-service-switcher.py b/examples/features/features-service-switcher.py index 9ded72a60..c2c78738a 100644 --- a/examples/features/features-service-switcher.py +++ b/examples/features/features-service-switcher.py @@ -94,19 +94,19 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): required=["location", "format"], ) - stt_cartesia = CartesiaSTTService(api_key=os.getenv("CARTESIA_API_KEY")) - stt_deepgram = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt_cartesia = CartesiaSTTService(api_key=os.environ["CARTESIA_API_KEY"]) + stt_deepgram = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) # Uses ServiceSwitcherStrategyManual by default stt_switcher = ServiceSwitcher(services=[stt_cartesia, stt_deepgram]) tts_cartesia = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) tts_deepgram = DeepgramTTSService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], settings=DeepgramTTSService.Settings( voice="aura-2-helena-en", ), @@ -117,11 +117,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): system_prompt = "You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way." llm_openai = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings(system_instruction=system_prompt), ) llm_google = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings(system_instruction=system_prompt), ) # Uses ServiceSwitcherStrategyManual by default diff --git a/examples/features/features-switch-languages.py b/examples/features/features-switch-languages.py index c47d66847..f99de0aec 100644 --- a/examples/features/features-switch-languages.py +++ b/examples/features/features-switch-languages.py @@ -42,14 +42,14 @@ class SwitchLanguage(ParallelPipeline): self._current_language = "English" english_tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) spanish_tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="d4db5fb9-f44b-4bd1-85fa-192e0f0d75f9", # Spanish-speaking Lady ), @@ -101,7 +101,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = DeepgramSTTService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], settings=DeepgramSTTService.Settings( language="multi", ), @@ -110,7 +110,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): tts = SwitchLanguage() llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You can speak the following languages: 'English' and 'Spanish'.", ), diff --git a/examples/features/features-switch-voices.py b/examples/features/features-switch-voices.py index 0c47d69ae..d33a10e8f 100644 --- a/examples/features/features-switch-voices.py +++ b/examples/features/features-switch-voices.py @@ -42,21 +42,21 @@ class SwitchVoices(ParallelPipeline): self._current_voice = "News Lady" news_lady = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="bf991597-6c13-47e4-8411-91ec2de5c466", # Newslady ), ) british_lady = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) barbershop_man = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="a0e99841-438c-4a64-b679-ae501e7d6091", # Barbershop Man ), @@ -114,12 +114,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = SwitchVoices() llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative and helpful way. You can do the following voices: 'News Lady', 'British Lady' and 'Barbershop Man'.", ), diff --git a/examples/features/features-user-email-gathering.py b/examples/features/features-user-email-gathering.py index 7e8a69011..f0ec3160f 100644 --- a/examples/features/features-user-email-gathering.py +++ b/examples/features/features-user-email-gathering.py @@ -60,13 +60,13 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) # Cartesia offers a `` tags that we can use to ask the user # to confirm the emails. # (see https://docs.cartesia.ai/build-with-sonic/formatting-text-for-sonic/spelling-out-input-text) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -84,7 +84,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You need to gather a valid email or emails from the user. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. If the user provides one or more email addresses confirm them with the user. Enclose all emails with tags, for example a@a.com.", ), diff --git a/examples/features/features-voicemail-detection.py b/examples/features/features-voicemail-detection.py index 854e546ec..4c8f07d6e 100644 --- a/examples/features/features-voicemail-detection.py +++ b/examples/features/features-voicemail-detection.py @@ -52,22 +52,22 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), ) - classifier_llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + classifier_llm = OpenAILLMService(api_key=os.environ["OPENAI_API_KEY"]) voicemail = VoicemailDetector(llm=classifier_llm) diff --git a/examples/features/features-wake-phrase.py b/examples/features/features-wake-phrase.py index 50f8b1fb8..8d39d5c60 100644 --- a/examples/features/features-wake-phrase.py +++ b/examples/features/features-wake-phrase.py @@ -57,21 +57,21 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = DeepgramSTTService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], settings=DeepgramSTTService.Settings( keyterm=["pipecat"], ), ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/function-calling/function-calling-anthropic-async-stream.py b/examples/function-calling/function-calling-anthropic-async-stream.py index 52069af68..b44bb5997 100644 --- a/examples/function-calling/function-calling-anthropic-async-stream.py +++ b/examples/function-calling/function-calling-anthropic-async-stream.py @@ -107,17 +107,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), + api_key=os.environ["ANTHROPIC_API_KEY"], enable_async_tool_cancellation=True, settings=AnthropicLLMService.Settings( system_instruction=( diff --git a/examples/function-calling/function-calling-anthropic-async.py b/examples/function-calling/function-calling-anthropic-async.py index 2c5fb9402..5046ec839 100644 --- a/examples/function-calling/function-calling-anthropic-async.py +++ b/examples/function-calling/function-calling-anthropic-async.py @@ -66,17 +66,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), + api_key=os.environ["ANTHROPIC_API_KEY"], enable_async_tool_cancellation=True, settings=AnthropicLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/function-calling/function-calling-anthropic-video.py b/examples/function-calling/function-calling-anthropic-video.py index c3a292f31..1b7617950 100644 --- a/examples/function-calling/function-calling-anthropic-video.py +++ b/examples/function-calling/function-calling-anthropic-video.py @@ -86,10 +86,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -97,7 +97,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Anthropic for vision analysis llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), + api_key=os.environ["ANTHROPIC_API_KEY"], settings=AnthropicLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You are able to describe images from the user camera.", ), diff --git a/examples/function-calling/function-calling-anthropic.py b/examples/function-calling/function-calling-anthropic.py index b8bb4eac6..3c4cf769a 100644 --- a/examples/function-calling/function-calling-anthropic.py +++ b/examples/function-calling/function-calling-anthropic.py @@ -65,17 +65,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), + api_key=os.environ["ANTHROPIC_API_KEY"], settings=AnthropicLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/function-calling/function-calling-aws-video.py b/examples/function-calling/function-calling-aws-video.py index 8db25671b..d6ae64438 100644 --- a/examples/function-calling/function-calling-aws-video.py +++ b/examples/function-calling/function-calling-aws-video.py @@ -86,10 +86,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), diff --git a/examples/function-calling/function-calling-azure.py b/examples/function-calling/function-calling-azure.py index 23c156da4..37d67b29b 100644 --- a/examples/function-calling/function-calling-azure.py +++ b/examples/function-calling/function-calling-azure.py @@ -60,18 +60,18 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = AzureLLMService( - api_key=os.getenv("AZURE_CHATGPT_API_KEY"), - endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + api_key=os.environ["AZURE_CHATGPT_API_KEY"], + endpoint=os.environ["AZURE_CHATGPT_ENDPOINT"], settings=AzureLLMService.Settings( model=os.getenv("AZURE_CHATGPT_MODEL"), system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/function-calling/function-calling-cerebras.py b/examples/function-calling/function-calling-cerebras.py index ad8475185..82d98d40e 100644 --- a/examples/function-calling/function-calling-cerebras.py +++ b/examples/function-calling/function-calling-cerebras.py @@ -60,17 +60,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = CerebrasLLMService( - api_key=os.getenv("CEREBRAS_API_KEY"), + api_key=os.environ["CEREBRAS_API_KEY"], settings=CerebrasLLMService.Settings( system_instruction="""You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. diff --git a/examples/function-calling/function-calling-deepseek.py b/examples/function-calling/function-calling-deepseek.py index f115299e2..fc354ccf5 100644 --- a/examples/function-calling/function-calling-deepseek.py +++ b/examples/function-calling/function-calling-deepseek.py @@ -60,17 +60,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = DeepSeekLLMService( - api_key=os.getenv("DEEPSEEK_API_KEY"), + api_key=os.environ["DEEPSEEK_API_KEY"], settings=DeepSeekLLMService.Settings( model="deepseek-chat", system_instruction="""You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. diff --git a/examples/function-calling/function-calling-direct.py b/examples/function-calling/function-calling-direct.py index 1c6ebe072..c885d6606 100644 --- a/examples/function-calling/function-calling-direct.py +++ b/examples/function-calling/function-calling-direct.py @@ -76,17 +76,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/function-calling/function-calling-fireworks.py b/examples/function-calling/function-calling-fireworks.py index 6ddb85981..8c681fa72 100644 --- a/examples/function-calling/function-calling-fireworks.py +++ b/examples/function-calling/function-calling-fireworks.py @@ -60,17 +60,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = FireworksLLMService( - api_key=os.getenv("FIREWORKS_API_KEY"), + api_key=os.environ["FIREWORKS_API_KEY"], settings=FireworksLLMService.Settings( model="accounts/fireworks/models/gpt-oss-20b", system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/function-calling/function-calling-google-async-stream.py b/examples/function-calling/function-calling-google-async-stream.py index 503880ca6..1660442b4 100644 --- a/examples/function-calling/function-calling-google-async-stream.py +++ b/examples/function-calling/function-calling-google-async-stream.py @@ -107,17 +107,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], enable_async_tool_cancellation=True, settings=GoogleLLMService.Settings( system_instruction=( diff --git a/examples/function-calling/function-calling-google-async.py b/examples/function-calling/function-calling-google-async.py index dbc86d664..f3c5fbc1d 100644 --- a/examples/function-calling/function-calling-google-async.py +++ b/examples/function-calling/function-calling-google-async.py @@ -98,10 +98,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -127,7 +127,7 @@ indicate you should use the get_image tool are: """ llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], enable_async_tool_cancellation=True, settings=GoogleLLMService.Settings( system_instruction=system_prompt, diff --git a/examples/function-calling/function-calling-google-vertex.py b/examples/function-calling/function-calling-google-vertex.py index 3bee2f1bb..290315f2c 100644 --- a/examples/function-calling/function-calling-google-vertex.py +++ b/examples/function-calling/function-calling-google-vertex.py @@ -60,19 +60,19 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = ElevenLabsTTSService( - api_key=os.getenv("ELEVENLABS_API_KEY", ""), + api_key=os.environ["ELEVENLABS_API_KEY"], settings=ElevenLabsTTSService.Settings( - voice=os.getenv("ELEVENLABS_VOICE_ID", ""), + voice=os.getenv("ELEVENLABS_VOICE_ID", "Xb7hH8MSUJpSbSDYk0k2"), ), ) llm = GoogleVertexLLMService( - credentials=os.getenv("GOOGLE_VERTEX_TEST_CREDENTIALS"), - project_id=os.getenv("GOOGLE_CLOUD_PROJECT_ID"), - location=os.getenv("GOOGLE_CLOUD_LOCATION"), + credentials=os.environ["GOOGLE_VERTEX_TEST_CREDENTIALS"], + project_id=os.environ["GOOGLE_CLOUD_PROJECT_ID"], + location=os.environ["GOOGLE_CLOUD_LOCATION"], settings=GoogleVertexLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), @@ -103,14 +103,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tools = ToolsSchema(standard_tools=[weather_function]) - messages = [ - { - "role": "developer", - "content": "Start a conversation with 'Hey there' to get the current weather.", - }, - ] - - context = LLMContext(messages, tools) + context = LLMContext(tools=tools) user_aggregator, assistant_aggregator = LLMContextAggregatorPair( context, user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), @@ -141,6 +134,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): async def on_client_connected(transport, client): logger.info(f"Client connected") # Kick off the conversation. + context.add_message( + { + "role": "developer", + "content": "Please introduce yourself to the user.", + } + ) await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") diff --git a/examples/function-calling/function-calling-google-video.py b/examples/function-calling/function-calling-google-video.py index 59816fdd5..65296c603 100644 --- a/examples/function-calling/function-calling-google-video.py +++ b/examples/function-calling/function-calling-google-video.py @@ -86,10 +86,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -97,7 +97,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Google Gemini model for vision analysis llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You are able to describe images from the user camera.", ), diff --git a/examples/function-calling/function-calling-google.py b/examples/function-calling/function-calling-google.py index 421a02a1e..1b6d5b152 100644 --- a/examples/function-calling/function-calling-google.py +++ b/examples/function-calling/function-calling-google.py @@ -96,10 +96,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -125,7 +125,7 @@ indicate you should use the get_image tool are: """ llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( system_instruction=system_prompt, ), diff --git a/examples/function-calling/function-calling-grok.py b/examples/function-calling/function-calling-grok.py index 05e1efaf9..537ab2025 100644 --- a/examples/function-calling/function-calling-grok.py +++ b/examples/function-calling/function-calling-grok.py @@ -62,10 +62,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = XAIHttpTTSService( - api_key=os.getenv("XAI_API_KEY"), + api_key=os.environ["XAI_API_KEY"], aiohttp_session=session, settings=XAIHttpTTSService.Settings( voice="eve", @@ -73,7 +73,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = GrokLLMService( - api_key=os.getenv("XAI_API_KEY"), + api_key=os.environ["XAI_API_KEY"], settings=GrokLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/function-calling/function-calling-groq.py b/examples/function-calling/function-calling-groq.py index 1e6e84338..d4c69349e 100644 --- a/examples/function-calling/function-calling-groq.py +++ b/examples/function-calling/function-calling-groq.py @@ -60,17 +60,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = GroqSTTService(api_key=os.getenv("GROQ_API_KEY")) + stt = GroqSTTService(api_key=os.environ["GROQ_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = GroqLLMService( - api_key=os.getenv("GROQ_API_KEY"), + api_key=os.environ["GROQ_API_KEY"], settings=GroqLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/function-calling/function-calling-mistral.py b/examples/function-calling/function-calling-mistral.py index c52d46c03..e44adf8ba 100644 --- a/examples/function-calling/function-calling-mistral.py +++ b/examples/function-calling/function-calling-mistral.py @@ -63,17 +63,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = MistralLLMService( - api_key=os.getenv("MISTRAL_API_KEY"), + api_key=os.environ["MISTRAL_API_KEY"], settings=MistralLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/function-calling/function-calling-moondream-video.py b/examples/function-calling/function-calling-moondream-video.py index b8076e7ce..56a0fa503 100644 --- a/examples/function-calling/function-calling-moondream-video.py +++ b/examples/function-calling/function-calling-moondream-video.py @@ -117,17 +117,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You are able to describe images from the user camera.", ), diff --git a/examples/function-calling/function-calling-nebius.py b/examples/function-calling/function-calling-nebius.py index 4fe9a378a..ed8fded2c 100644 --- a/examples/function-calling/function-calling-nebius.py +++ b/examples/function-calling/function-calling-nebius.py @@ -63,17 +63,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = NebiusLLMService( - api_key=os.getenv("NEBIUS_API_KEY"), + api_key=os.environ["NEBIUS_API_KEY"], settings=NebiusLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/function-calling/function-calling-novita.py b/examples/function-calling/function-calling-novita.py index da60d890b..d9838e37f 100644 --- a/examples/function-calling/function-calling-novita.py +++ b/examples/function-calling/function-calling-novita.py @@ -63,17 +63,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = NovitaLLMService( - api_key=os.getenv("NOVITA_API_KEY"), + api_key=os.environ["NOVITA_API_KEY"], settings=NovitaLLMService.Settings( model="openai/gpt-oss-120b", system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/function-calling/function-calling-nvidia.py b/examples/function-calling/function-calling-nvidia.py index e3db6db8d..2b4818ceb 100644 --- a/examples/function-calling/function-calling-nvidia.py +++ b/examples/function-calling/function-calling-nvidia.py @@ -60,17 +60,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = NvidiaLLMService( - api_key=os.getenv("NVIDIA_API_KEY"), + api_key=os.environ["NVIDIA_API_KEY"], settings=NvidiaLLMService.Settings( model="nvidia/llama-3.3-nemotron-super-49b-v1.5", # Recommended when turning thinking off diff --git a/examples/function-calling/function-calling-ollama.py b/examples/function-calling/function-calling-ollama.py index 87491f3d6..ec6c3948a 100644 --- a/examples/function-calling/function-calling-ollama.py +++ b/examples/function-calling/function-calling-ollama.py @@ -64,10 +64,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), diff --git a/examples/function-calling/function-calling-openai-async-stream.py b/examples/function-calling/function-calling-openai-async-stream.py index a60a5198f..b1d9d724b 100644 --- a/examples/function-calling/function-calling-openai-async-stream.py +++ b/examples/function-calling/function-calling-openai-async-stream.py @@ -107,17 +107,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], enable_async_tool_cancellation=True, settings=OpenAILLMService.Settings( system_instruction=( diff --git a/examples/function-calling/function-calling-openai-async.py b/examples/function-calling/function-calling-openai-async.py index 31c932d5f..47039711f 100644 --- a/examples/function-calling/function-calling-openai-async.py +++ b/examples/function-calling/function-calling-openai-async.py @@ -70,7 +70,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = OpenAISTTService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAISTTService.Settings( model="gpt-4o-transcribe", prompt="Expect words related weather, such as temperature and conditions. And restaurant names.", @@ -78,7 +78,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = OpenAITTSService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAITTSService.Settings( voice="ballad", ), @@ -86,7 +86,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], enable_async_tool_cancellation=True, settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/function-calling/function-calling-openai-responses-async-stream.py b/examples/function-calling/function-calling-openai-responses-async-stream.py index 745c18ae1..4007c21eb 100644 --- a/examples/function-calling/function-calling-openai-responses-async-stream.py +++ b/examples/function-calling/function-calling-openai-responses-async-stream.py @@ -107,17 +107,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], enable_async_tool_cancellation=True, settings=OpenAIResponsesLLMService.Settings( system_instruction=( diff --git a/examples/function-calling/function-calling-openai-responses-async.py b/examples/function-calling/function-calling-openai-responses-async.py index 368f41fe1..f7e6d370b 100644 --- a/examples/function-calling/function-calling-openai-responses-async.py +++ b/examples/function-calling/function-calling-openai-responses-async.py @@ -66,17 +66,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], enable_async_tool_cancellation=True, settings=OpenAIResponsesLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/function-calling/function-calling-openai-responses-http.py b/examples/function-calling/function-calling-openai-responses-http.py index ae38c98f5..e27d5d82b 100644 --- a/examples/function-calling/function-calling-openai-responses-http.py +++ b/examples/function-calling/function-calling-openai-responses-http.py @@ -63,17 +63,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesHttpLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIResponsesHttpLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/function-calling/function-calling-openai-responses-video-http.py b/examples/function-calling/function-calling-openai-responses-video-http.py index ad83aab24..9d9953359 100644 --- a/examples/function-calling/function-calling-openai-responses-video-http.py +++ b/examples/function-calling/function-calling-openai-responses-video-http.py @@ -87,17 +87,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesHttpLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIResponsesHttpLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You are able to describe images from the user camera.", ), diff --git a/examples/function-calling/function-calling-openai-responses-video.py b/examples/function-calling/function-calling-openai-responses-video.py index 9a77af400..3cb159a3f 100644 --- a/examples/function-calling/function-calling-openai-responses-video.py +++ b/examples/function-calling/function-calling-openai-responses-video.py @@ -87,17 +87,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIResponsesLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You are able to describe images from the user camera.", ), diff --git a/examples/function-calling/function-calling-openai-responses.py b/examples/function-calling/function-calling-openai-responses.py index 171e7b36e..97481d01e 100644 --- a/examples/function-calling/function-calling-openai-responses.py +++ b/examples/function-calling/function-calling-openai-responses.py @@ -63,17 +63,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIResponsesLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/function-calling/function-calling-openai-video.py b/examples/function-calling/function-calling-openai-video.py index 3c4fcd2f6..8ca7725b0 100644 --- a/examples/function-calling/function-calling-openai-video.py +++ b/examples/function-calling/function-calling-openai-video.py @@ -87,17 +87,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You are able to describe images from the user camera.", ), diff --git a/examples/function-calling/function-calling-openai.py b/examples/function-calling/function-calling-openai.py index 2b59d7072..2a9779742 100644 --- a/examples/function-calling/function-calling-openai.py +++ b/examples/function-calling/function-calling-openai.py @@ -64,7 +64,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = OpenAISTTService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAISTTService.Settings( model="gpt-4o-transcribe", prompt="Expect words related weather, such as temperature and conditions. And restaurant names.", @@ -72,7 +72,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = OpenAITTSService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAITTSService.Settings( voice="ballad", ), @@ -80,7 +80,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/function-calling/function-calling-openrouter.py b/examples/function-calling/function-calling-openrouter.py index b341ca71c..299846133 100644 --- a/examples/function-calling/function-calling-openrouter.py +++ b/examples/function-calling/function-calling-openrouter.py @@ -60,11 +60,11 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = AzureTTSService( - api_key=os.getenv("AZURE_SPEECH_API_KEY"), - region=os.getenv("AZURE_SPEECH_REGION"), + api_key=os.environ["AZURE_SPEECH_API_KEY"], + region=os.environ["AZURE_SPEECH_REGION"], settings=AzureTTSService.Settings( voice="en-US-JennyNeural", language="en-US", @@ -74,7 +74,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenRouterLLMService( - api_key=os.getenv("OPENROUTER_API_KEY"), + api_key=os.environ["OPENROUTER_API_KEY"], settings=OpenRouterLLMService.Settings( model="openai/gpt-4o-2024-11-20", system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/function-calling/function-calling-perplexity.py b/examples/function-calling/function-calling-perplexity.py index 0fa3ff928..8abbc3618 100644 --- a/examples/function-calling/function-calling-perplexity.py +++ b/examples/function-calling/function-calling-perplexity.py @@ -58,17 +58,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = PerplexityLLMService( - api_key=os.getenv("PERPLEXITY_API_KEY"), + api_key=os.environ["PERPLEXITY_API_KEY"], settings=PerplexityLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/function-calling/function-calling-qwen.py b/examples/function-calling/function-calling-qwen.py index 9a6bf6d8b..0bbb7311d 100644 --- a/examples/function-calling/function-calling-qwen.py +++ b/examples/function-calling/function-calling-qwen.py @@ -60,17 +60,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = QwenLLMService( - api_key=os.getenv("QWEN_API_KEY"), + api_key=os.environ["QWEN_API_KEY"], model="qwen2.5-72b-instruct", settings=QwenLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/function-calling/function-calling-sambanova.py b/examples/function-calling/function-calling-sambanova.py index c1e880bc0..528c4fc47 100644 --- a/examples/function-calling/function-calling-sambanova.py +++ b/examples/function-calling/function-calling-sambanova.py @@ -61,18 +61,18 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = DeepgramSTTService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = SambaNovaLLMService( - api_key=os.getenv("SAMBANOVA_API_KEY"), + api_key=os.environ["SAMBANOVA_API_KEY"], settings=SambaNovaLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/function-calling/function-calling-sarvam.py b/examples/function-calling/function-calling-sarvam.py index bd3fa42fb..830860a9a 100644 --- a/examples/function-calling/function-calling-sarvam.py +++ b/examples/function-calling/function-calling-sarvam.py @@ -64,21 +64,21 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = SarvamSTTService( - api_key=os.getenv("SARVAM_API_KEY"), + api_key=os.environ["SARVAM_API_KEY"], settings=SarvamSTTService.Settings( model="saaras:v3", ), ) tts = SarvamTTSService( - api_key=os.getenv("SARVAM_API_KEY"), + api_key=os.environ["SARVAM_API_KEY"], settings=SarvamTTSService.Settings( model="bulbul:v3", voice="shubh", ), ) llm = SarvamLLMService( - api_key=os.getenv("SARVAM_API_KEY"), + api_key=os.environ["SARVAM_API_KEY"], settings=SarvamLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/function-calling/function-calling-together.py b/examples/function-calling/function-calling-together.py index 78da7fbe9..e785588f4 100644 --- a/examples/function-calling/function-calling-together.py +++ b/examples/function-calling/function-calling-together.py @@ -60,17 +60,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = TogetherLLMService( - api_key=os.getenv("TOGETHER_API_KEY"), + api_key=os.environ["TOGETHER_API_KEY"], settings=TogetherLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/getting-started/01-say-one-thing.py b/examples/getting-started/01-say-one-thing.py index fdfecde6c..1c6e1c703 100644 --- a/examples/getting-started/01-say-one-thing.py +++ b/examples/getting-started/01-say-one-thing.py @@ -36,7 +36,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), diff --git a/examples/getting-started/01a-local-audio.py b/examples/getting-started/01a-local-audio.py index 1e18b4b03..02a489c93 100644 --- a/examples/getting-started/01a-local-audio.py +++ b/examples/getting-started/01a-local-audio.py @@ -28,7 +28,7 @@ async def main(): transport = LocalAudioTransport(LocalAudioTransportParams(audio_out_enabled=True)) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), diff --git a/examples/getting-started/02-llm-say-one-thing.py b/examples/getting-started/02-llm-say-one-thing.py index 74809ed2a..df52772cf 100644 --- a/examples/getting-started/02-llm-say-one-thing.py +++ b/examples/getting-started/02-llm-say-one-thing.py @@ -38,14 +38,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/getting-started/03-still-frame.py b/examples/getting-started/03-still-frame.py index 259072959..61b211499 100644 --- a/examples/getting-started/03-still-frame.py +++ b/examples/getting-started/03-still-frame.py @@ -42,7 +42,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") imagegen = GoogleImageGenService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], ) task = PipelineTask( diff --git a/examples/getting-started/04-sync-speech-and-image.py b/examples/getting-started/04-sync-speech-and-image.py index f0e2ff9c7..c26bb324e 100644 --- a/examples/getting-started/04-sync-speech-and-image.py +++ b/examples/getting-started/04-sync-speech-and-image.py @@ -108,10 +108,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Create an HTTP session for API calls async with aiohttp.ClientSession() as session: - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + llm = OpenAILLMService(api_key=os.environ["OPENAI_API_KEY"]) tts = CartesiaHttpTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaHttpTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), diff --git a/examples/getting-started/05-speaking-state.py b/examples/getting-started/05-speaking-state.py index 36762f6b0..32206d1b0 100644 --- a/examples/getting-started/05-speaking-state.py +++ b/examples/getting-started/05-speaking-state.py @@ -96,17 +96,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/getting-started/06-voice-agent.py b/examples/getting-started/06-voice-agent.py index 057429589..c9fe1cf12 100644 --- a/examples/getting-started/06-voice-agent.py +++ b/examples/getting-started/06-voice-agent.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/getting-started/06a-voice-agent-local.py b/examples/getting-started/06a-voice-agent-local.py index 3d183b405..8864f7aff 100644 --- a/examples/getting-started/06a-voice-agent-local.py +++ b/examples/getting-started/06a-voice-agent-local.py @@ -40,17 +40,17 @@ async def main(): ) ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/getting-started/07-function-calling.py b/examples/getting-started/07-function-calling.py index 085937da9..173f6a6fe 100644 --- a/examples/getting-started/07-function-calling.py +++ b/examples/getting-started/07-function-calling.py @@ -63,17 +63,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/mcp/mcp-multiple-mcp.py b/examples/mcp/mcp-multiple-mcp.py index 7bdfa1343..8d7717291 100644 --- a/examples/mcp/mcp-multiple-mcp.py +++ b/examples/mcp/mcp-multiple-mcp.py @@ -53,10 +53,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -77,7 +77,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): """ llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), + api_key=os.environ["ANTHROPIC_API_KEY"], settings=AnthropicLLMService.Settings( system_instruction=system_prompt, ), diff --git a/examples/mcp/mcp-stdio.py b/examples/mcp/mcp-stdio.py index daae45c0c..a93463ede 100644 --- a/examples/mcp/mcp-stdio.py +++ b/examples/mcp/mcp-stdio.py @@ -49,10 +49,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -71,7 +71,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): """ llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), + api_key=os.environ["ANTHROPIC_API_KEY"], settings=AnthropicLLMService.Settings( system_instruction=system_prompt, ), diff --git a/examples/mcp/mcp-streamable-http-gemini-live.py b/examples/mcp/mcp-streamable-http-gemini-live.py index 8b824454f..213667e7a 100644 --- a/examples/mcp/mcp-streamable-http-gemini-live.py +++ b/examples/mcp/mcp-streamable-http-gemini-live.py @@ -23,8 +23,6 @@ from pipecat.processors.aggregators.llm_response_universal import ( ) from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService from pipecat.services.mcp_service import MCPClient from pipecat.transports.base_transport import BaseTransport, TransportParams @@ -54,15 +52,6 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - settings=CartesiaTTSService.Settings( - voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ), - ) - system = f""" You are a helpful LLM in a voice call. Your goal is to answer questions about the user's GitHub repositories and account. @@ -85,7 +74,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): tools = await mcp.get_tools_schema() llm = GeminiLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], system_instruction=system, tools=tools, ) diff --git a/examples/mcp/mcp-streamable-http.py b/examples/mcp/mcp-streamable-http.py index 59860d307..b4823c649 100644 --- a/examples/mcp/mcp-streamable-http.py +++ b/examples/mcp/mcp-streamable-http.py @@ -54,10 +54,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -73,7 +73,7 @@ Just respond with short sentences when you are carrying out tool calls. """ llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( system_instruction=system_prompt, ), diff --git a/examples/observability/observability-observer.py b/examples/observability/observability-observer.py index 457ba7bb4..4a1bbbbfa 100644 --- a/examples/observability/observability-observer.py +++ b/examples/observability/observability-observer.py @@ -100,17 +100,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/observability/observability-sentry-metrics.py b/examples/observability/observability-sentry-metrics.py index 3d4146150..9a8f8d553 100644 --- a/examples/observability/observability-sentry-metrics.py +++ b/examples/observability/observability-sentry-metrics.py @@ -60,12 +60,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) stt = DeepgramSTTService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], metrics=SentryMetrics(), ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -73,7 +73,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], metrics=SentryMetrics(), settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/persistent-context/persistent-context-anthropic.py b/examples/persistent-context/persistent-context-anthropic.py index 791f00440..cdddc9080 100644 --- a/examples/persistent-context/persistent-context-anthropic.py +++ b/examples/persistent-context/persistent-context-anthropic.py @@ -170,17 +170,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), + api_key=os.environ["ANTHROPIC_API_KEY"], settings=AnthropicLLMService.Settings( system_instruction=system_instruction, ), diff --git a/examples/persistent-context/persistent-context-aws-nova-sonic.py b/examples/persistent-context/persistent-context-aws-nova-sonic.py index 62b9e7eab..8a4b7ba47 100644 --- a/examples/persistent-context/persistent-context-aws-nova-sonic.py +++ b/examples/persistent-context/persistent-context-aws-nova-sonic.py @@ -87,7 +87,9 @@ async def save_conversation(params: FunctionCallParams): # the simplest thing to do is to pop messages until the last one is an assistant # response while messages and not ( - messages[-1].get("role") == "assistant" and "content" in messages[-1] + isinstance(messages[-1], dict) + and messages[-1].get("role") == "assistant" + and "content" in messages[-1] ): messages.pop() if messages: # we never expect this to be empty @@ -125,6 +127,7 @@ async def load_conversation(params: FunctionCallParams): } ) params.context.set_messages(messages) + assert isinstance(params.llm, AWSNovaSonicLLMService) await params.llm.reset_conversation() # await params.llm.trigger_assistant_response() except Exception as e: @@ -219,9 +222,9 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = AWSNovaSonicLLMService( - secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), - access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), - region=os.getenv("AWS_REGION"), # as of 2025-05-06, us-east-1 is the only supported region + secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"], + access_key_id=os.environ["AWS_ACCESS_KEY_ID"], + region=os.environ["AWS_REGION"], # as of 2025-05-06, us-east-1 is the only supported region settings=AWSNovaSonicLLMService.Settings( voice="tiffany", # matthew, tiffany, amy system_instruction=system_instruction, diff --git a/examples/persistent-context/persistent-context-gemini.py b/examples/persistent-context/persistent-context-gemini.py index 2ec3cd0bb..2ae43bf87 100644 --- a/examples/persistent-context/persistent-context-gemini.py +++ b/examples/persistent-context/persistent-context-gemini.py @@ -243,17 +243,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], system_instruction=system_instruction, ) diff --git a/examples/persistent-context/persistent-context-grok-realtime.py b/examples/persistent-context/persistent-context-grok-realtime.py index 4a990d24e..e21dbf1c3 100644 --- a/examples/persistent-context/persistent-context-grok-realtime.py +++ b/examples/persistent-context/persistent-context-grok-realtime.py @@ -192,7 +192,7 @@ Remember, your responses should be short - just one or two sentences usually.""" ) llm = GrokRealtimeLLMService( - api_key=os.getenv("XAI_API_KEY"), + api_key=os.environ["XAI_API_KEY"], session_properties=session_properties, ) diff --git a/examples/persistent-context/persistent-context-openai-realtime.py b/examples/persistent-context/persistent-context-openai-realtime.py index 0a7a888a8..067e1f73f 100644 --- a/examples/persistent-context/persistent-context-openai-realtime.py +++ b/examples/persistent-context/persistent-context-openai-realtime.py @@ -93,6 +93,7 @@ async def load_conversation(params: FunctionCallParams): try: with open(filename) as file: params.context.set_messages(json.load(file)) + assert isinstance(params.llm, OpenAIRealtimeLLMService) await params.llm.reset_conversation() # NOTE: we manually create a response here rather than relying # on the function callback to trigger one since we've reset the @@ -171,10 +172,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) llm = OpenAIRealtimeLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIRealtimeLLMService.Settings( system_instruction="""Your knowledge cutoff is 2023-10. You are a helpful and friendly AI. diff --git a/examples/persistent-context/persistent-context-openai-responses-http.py b/examples/persistent-context/persistent-context-openai-responses-http.py index d77513049..362fd86f6 100644 --- a/examples/persistent-context/persistent-context-openai-responses-http.py +++ b/examples/persistent-context/persistent-context-openai-responses-http.py @@ -171,17 +171,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesHttpLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIResponsesHttpLLMService.Settings( system_instruction=system_instruction, ), diff --git a/examples/persistent-context/persistent-context-openai-responses.py b/examples/persistent-context/persistent-context-openai-responses.py index c89fe3ff1..9c3f33c94 100644 --- a/examples/persistent-context/persistent-context-openai-responses.py +++ b/examples/persistent-context/persistent-context-openai-responses.py @@ -171,17 +171,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIResponsesLLMService.Settings( system_instruction=system_instruction, ), diff --git a/examples/persistent-context/persistent-context-openai.py b/examples/persistent-context/persistent-context-openai.py index 22e160102..fa29ad5c2 100644 --- a/examples/persistent-context/persistent-context-openai.py +++ b/examples/persistent-context/persistent-context-openai.py @@ -171,17 +171,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], system_instruction=system_instruction, ) diff --git a/examples/rag/rag-gemini-grounding-metadata.py b/examples/rag/rag-gemini-grounding-metadata.py index 5b706d6c9..cddefe591 100644 --- a/examples/rag/rag-gemini-grounding-metadata.py +++ b/examples/rag/rag-gemini-grounding-metadata.py @@ -95,10 +95,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -106,7 +106,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Initialize the Gemini Multimodal Live model llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( system_instruction=system_instruction, ), diff --git a/examples/rag/rag-gemini.py b/examples/rag/rag-gemini.py index 0f9a8ddb2..1567ad051 100644 --- a/examples/rag/rag-gemini.py +++ b/examples/rag/rag-gemini.py @@ -52,7 +52,7 @@ import os import time from dotenv import load_dotenv -from google import genai +from google import genai # pyright: ignore[reportAttributeAccessIssue] from loguru import logger from pipecat.adapters.schemas.function_schema import FunctionSchema @@ -179,10 +179,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="f9836c6e-a0bd-460e-9d3c-f7299fa60f94", # Southern Lady ), @@ -197,7 +197,7 @@ Your response will be turned into speech so use only simple words and punctuatio """ llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( model=VOICE_MODEL, system_instruction=system_prompt, diff --git a/examples/rag/rag-mem0.py b/examples/rag/rag-mem0.py index c7c0f13bd..1cbf97d14 100644 --- a/examples/rag/rag-mem0.py +++ b/examples/rag/rag-mem0.py @@ -133,11 +133,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) # Initialize text-to-speech service tts = ElevenLabsTTSService( - api_key=os.getenv("ELEVENLABS_API_KEY"), + api_key=os.environ["ELEVENLABS_API_KEY"], settings=ElevenLabsTTSService.Settings( voice="pNInz6obpgDQGcFmaJgB", ), @@ -196,7 +196,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Initialize LLM service llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="""You are a personal assistant. You can remember things about the person you are talking to. Some Guidelines: diff --git a/examples/realtime/realtime-aws-nova-sonic.py b/examples/realtime/realtime-aws-nova-sonic.py index f3a1a0737..5e6a2037c 100644 --- a/examples/realtime/realtime-aws-nova-sonic.py +++ b/examples/realtime/realtime-aws-nova-sonic.py @@ -116,8 +116,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Create the AWS Nova Sonic LLM service llm = AWSNovaSonicLLMService( - secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), - access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), + secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"], + access_key_id=os.environ["AWS_ACCESS_KEY_ID"], # as of 2025-12-09, these are the supported regions: # - Nova 2 Sonic (the default model): # - us-east-1 @@ -126,7 +126,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # - Nova Sonic (the older model): # - us-east-1 # - ap-northeast-1 - region=os.getenv("AWS_REGION"), + region=os.environ["AWS_REGION"], session_token=os.getenv("AWS_SESSION_TOKEN"), settings=AWSNovaSonicLLMService.Settings( voice="tiffany", diff --git a/examples/realtime/realtime-azure.py b/examples/realtime/realtime-azure.py index c57602f6f..c4ff80dcb 100644 --- a/examples/realtime/realtime-azure.py +++ b/examples/realtime/realtime-azure.py @@ -112,8 +112,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") llm = AzureRealtimeLLMService( - api_key=os.getenv("AZURE_REALTIME_API_KEY"), - base_url=os.getenv("AZURE_REALTIME_BASE_URL"), + api_key=os.environ["AZURE_REALTIME_API_KEY"], + base_url=os.environ["AZURE_REALTIME_BASE_URL"], settings=AzureRealtimeLLMService.Settings( system_instruction="""You are a helpful and friendly AI. diff --git a/examples/realtime/realtime-gemini-live-files-api.py b/examples/realtime/realtime-gemini-live-files-api.py index 1871bb5bf..99f35d52a 100644 --- a/examples/realtime/realtime-gemini-live-files-api.py +++ b/examples/realtime/realtime-gemini-live-files-api.py @@ -104,7 +104,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Initialize Gemini service with File API support llm = GeminiLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GeminiLiveLLMService.Settings( system_instruction=system_instruction, voice="Charon", # Aoede, Charon, Fenrir, Kore, Puck diff --git a/examples/realtime/realtime-gemini-live-function-calling.py b/examples/realtime/realtime-gemini-live-function-calling.py index a370fdf57..d038a5c4f 100644 --- a/examples/realtime/realtime-gemini-live-function-calling.py +++ b/examples/realtime/realtime-gemini-live-function-calling.py @@ -114,7 +114,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = GeminiLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GeminiLiveLLMService.Settings( system_instruction=system_instruction, ), diff --git a/examples/realtime/realtime-gemini-live-google-search.py b/examples/realtime/realtime-gemini-live-google-search.py index 7822bb085..6b132cef1 100644 --- a/examples/realtime/realtime-gemini-live-google-search.py +++ b/examples/realtime/realtime-gemini-live-google-search.py @@ -67,7 +67,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Initialize the Gemini Multimodal Live model llm = GeminiLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GeminiLiveLLMService.Settings( voice="Puck", # Aoede, Charon, Fenrir, Kore, Puck system_instruction=system_instruction, diff --git a/examples/realtime/realtime-gemini-live-graceful-end.py b/examples/realtime/realtime-gemini-live-graceful-end.py index 7eca04fb3..d65414022 100644 --- a/examples/realtime/realtime-gemini-live-graceful-end.py +++ b/examples/realtime/realtime-gemini-live-graceful-end.py @@ -133,7 +133,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = GeminiLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GeminiLiveLLMService.Settings( system_instruction=system_instruction, ), diff --git a/examples/realtime/realtime-gemini-live-grounding-metadata.py b/examples/realtime/realtime-gemini-live-grounding-metadata.py index 31fd7d832..e78d5cffa 100644 --- a/examples/realtime/realtime-gemini-live-grounding-metadata.py +++ b/examples/realtime/realtime-gemini-live-grounding-metadata.py @@ -75,7 +75,8 @@ class GroundingMetadataProcessor(FrameProcessor): if isinstance(frame, LLMSearchResponseFrame): self._grounding_count += 1 logger.info(f"\n\n🔍 GROUNDING METADATA RECEIVED #{self._grounding_count}\n") - logger.info(f"📝 Search Result Text: {frame.search_result[:200]}...") + if frame.search_result: + logger.info(f"📝 Search Result Text: {frame.search_result[:200]}...") if frame.rendered_content: logger.info(f"🔗 Rendered Content: {frame.rendered_content}") @@ -101,7 +102,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = GeminiLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GeminiLiveLLMService.Settings( system_instruction=SYSTEM_INSTRUCTION, voice="Charon", # Aoede, Charon, Fenrir, Kore, Puck @@ -111,16 +112,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Create a processor to capture grounding metadata grounding_processor = GroundingMetadataProcessor() - - messages = [ - { - "role": "user", - "content": "Please introduce yourself and let me know that you can help with current information by searching the web. Ask me what current information I'd like to know about.", - }, - ] - # Set up conversation context and management - context = LLMContext(messages) + context = LLMContext() # Server-side VAD is enabled by default; no local VAD is added. user_aggregator, assistant_aggregator = LLMContextAggregatorPair(context) @@ -144,6 +137,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): async def on_client_connected(transport, client): logger.info(f"Client connected") # Kick off the conversation. + context.add_message( + { + "role": "developer", + "content": "Please introduce yourself and let me know that you can help with current information by searching the web. Ask me what current information I'd like to know about.", + } + ) await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") diff --git a/examples/realtime/realtime-gemini-live-local-vad.py b/examples/realtime/realtime-gemini-live-local-vad.py index 7a73742e9..c7580f0c6 100644 --- a/examples/realtime/realtime-gemini-live-local-vad.py +++ b/examples/realtime/realtime-gemini-live-local-vad.py @@ -54,7 +54,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") llm = GeminiLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GeminiLiveLLMService.Settings( voice="Aoede", # Puck, Charon, Kore, Fenrir, Aoede vad=GeminiVADParams(disabled=True), diff --git a/examples/realtime/realtime-gemini-live-vertex-function-calling.py b/examples/realtime/realtime-gemini-live-vertex-function-calling.py index 68add5bc9..2c0861006 100644 --- a/examples/realtime/realtime-gemini-live-vertex-function-calling.py +++ b/examples/realtime/realtime-gemini-live-vertex-function-calling.py @@ -110,8 +110,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = GeminiLiveVertexLLMService( credentials=os.getenv("GOOGLE_VERTEX_TEST_CREDENTIALS"), - project_id=os.getenv("GOOGLE_CLOUD_PROJECT_ID"), - location=os.getenv("GOOGLE_CLOUD_LOCATION"), + project_id=os.environ["GOOGLE_CLOUD_PROJECT_ID"], + location=os.environ["GOOGLE_CLOUD_LOCATION"], settings=GeminiLiveVertexLLMService.Settings( system_instruction=system_instruction, voice="Puck", # Aoede, Charon, Fenrir, Kore, Puck diff --git a/examples/realtime/realtime-gemini-live-video.py b/examples/realtime/realtime-gemini-live-video.py index 269f22718..4319b534a 100644 --- a/examples/realtime/realtime-gemini-live-video.py +++ b/examples/realtime/realtime-gemini-live-video.py @@ -47,7 +47,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = GeminiLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GeminiLiveLLMService.Settings( voice="Aoede", # Puck, Charon, Kore, Fenrir, Aoede # system_instruction="Talk like a pirate." diff --git a/examples/realtime/realtime-gemini-live.py b/examples/realtime/realtime-gemini-live.py index dc9e8e55a..04fa9c625 100644 --- a/examples/realtime/realtime-gemini-live.py +++ b/examples/realtime/realtime-gemini-live.py @@ -52,7 +52,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") llm = GeminiLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GeminiLiveLLMService.Settings( voice="Aoede", # Puck, Charon, Kore, Fenrir, Aoede # system_instruction="Talk like a pirate." diff --git a/examples/realtime/realtime-grok.py b/examples/realtime/realtime-grok.py index 3f1543871..1e122865a 100644 --- a/examples/realtime/realtime-grok.py +++ b/examples/realtime/realtime-grok.py @@ -179,7 +179,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Create the Grok Realtime LLM service llm = GrokRealtimeLLMService( - api_key=os.getenv("XAI_API_KEY"), + api_key=os.environ["XAI_API_KEY"], settings=GrokRealtimeLLMService.Settings( system_instruction="""You are a helpful and friendly AI assistant powered by Grok. diff --git a/examples/realtime/realtime-inworld.py b/examples/realtime/realtime-inworld.py index 67bed6df9..76defbc24 100644 --- a/examples/realtime/realtime-inworld.py +++ b/examples/realtime/realtime-inworld.py @@ -84,7 +84,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # llm_model can be any supported model or an Inworld Router. # See: https://docs.inworld.ai/router/introduction llm = InworldRealtimeLLMService( - api_key=os.getenv("INWORLD_API_KEY"), + api_key=os.environ["INWORLD_API_KEY"], llm_model="xai/grok-4-1-fast-non-reasoning", voice="Sarah", settings=InworldRealtimeLLMService.Settings( diff --git a/examples/realtime/realtime-openai-live-video.py b/examples/realtime/realtime-openai-live-video.py index fdf7c04e8..b35aa2648 100644 --- a/examples/realtime/realtime-openai-live-video.py +++ b/examples/realtime/realtime-openai-live-video.py @@ -62,7 +62,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") llm = OpenAIRealtimeLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIRealtimeLLMService.Settings( system_instruction="""You are a helpful and friendly AI. @@ -133,8 +133,8 @@ Remember, your responses should be short. Just one or two sentences, usually. Re async def on_client_connected(transport, client): logger.info(f"Client connected: {client}") - await maybe_capture_participant_camera(transport, client, framerate=0.5) - await maybe_capture_participant_screen(transport, client, framerate=0.5) + await maybe_capture_participant_camera(transport, client, framerate=1) + await maybe_capture_participant_screen(transport, client, framerate=1) await task.queue_frames([LLMRunFrame()]) diff --git a/examples/realtime/realtime-openai-text.py b/examples/realtime/realtime-openai-text.py index 4147ca4c3..f36a3c773 100644 --- a/examples/realtime/realtime-openai-text.py +++ b/examples/realtime/realtime-openai-text.py @@ -117,7 +117,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") llm = OpenAIRealtimeLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIRealtimeLLMSettings( system_instruction="""You are a helpful and friendly AI. @@ -156,7 +156,7 @@ Remember, your responses should be short. Just one or two sentences, usually. Re ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), diff --git a/examples/realtime/realtime-openai.py b/examples/realtime/realtime-openai.py index 9c47bac77..6f347ed9b 100644 --- a/examples/realtime/realtime-openai.py +++ b/examples/realtime/realtime-openai.py @@ -136,7 +136,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") llm = OpenAIRealtimeLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIRealtimeLLMService.Settings( system_instruction="""You are a helpful and friendly AI. diff --git a/examples/realtime/realtime-ultravox-text.py b/examples/realtime/realtime-ultravox-text.py index 8b876048a..a144af919 100644 --- a/examples/realtime/realtime-ultravox-text.py +++ b/examples/realtime/realtime-ultravox-text.py @@ -168,7 +168,7 @@ There is also a secret menu that changes daily. If the user asks about it, use t llm = UltravoxRealtimeLLMService( params=OneShotInputParams( - api_key=os.getenv("ULTRAVOX_API_KEY"), + api_key=os.environ["ULTRAVOX_API_KEY"], system_prompt=system_prompt, temperature=0.3, max_duration=datetime.timedelta(minutes=3), diff --git a/examples/realtime/realtime-ultravox.py b/examples/realtime/realtime-ultravox.py index 7aaacdeaf..ed46aba2f 100644 --- a/examples/realtime/realtime-ultravox.py +++ b/examples/realtime/realtime-ultravox.py @@ -166,7 +166,7 @@ There is also a secret menu that changes daily. If the user asks about it, use t llm = UltravoxRealtimeLLMService( params=OneShotInputParams( - api_key=os.getenv("ULTRAVOX_API_KEY"), + api_key=os.environ["ULTRAVOX_API_KEY"], system_prompt=system_prompt, temperature=0.3, max_duration=datetime.timedelta(minutes=3), diff --git a/examples/thinking/thinking-anthropic.py b/examples/thinking/thinking-anthropic.py index 0cd768925..fe64aa3f3 100644 --- a/examples/thinking/thinking-anthropic.py +++ b/examples/thinking/thinking-anthropic.py @@ -52,17 +52,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), + api_key=os.environ["ANTHROPIC_API_KEY"], settings=AnthropicLLMService.Settings( thinking=AnthropicLLMService.ThinkingConfig( type="enabled", diff --git a/examples/thinking/thinking-functions-anthropic.py b/examples/thinking/thinking-functions-anthropic.py index b62a96452..f69b271fb 100644 --- a/examples/thinking/thinking-functions-anthropic.py +++ b/examples/thinking/thinking-functions-anthropic.py @@ -73,17 +73,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), + api_key=os.environ["ANTHROPIC_API_KEY"], settings=AnthropicLLMService.Settings( thinking=AnthropicLLMService.ThinkingConfig( type="enabled", diff --git a/examples/thinking/thinking-functions-google.py b/examples/thinking/thinking-functions-google.py index e06a0195b..680f2c70b 100644 --- a/examples/thinking/thinking-functions-google.py +++ b/examples/thinking/thinking-functions-google.py @@ -73,17 +73,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], # model="gemini-3-pro-preview", # A more powerful reasoning model, but slower settings=GoogleLLMService.Settings( thinking=GoogleLLMService.ThinkingConfig( diff --git a/examples/thinking/thinking-google.py b/examples/thinking/thinking-google.py index 94d3d28eb..12de4a313 100644 --- a/examples/thinking/thinking-google.py +++ b/examples/thinking/thinking-google.py @@ -52,17 +52,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], # model="gemini-3-pro-preview", # A more powerful reasoning model, but slower settings=GoogleLLMService.Settings( thinking=GoogleLLMService.ThinkingConfig( diff --git a/examples/transcription/transcription-assemblyai.py b/examples/transcription/transcription-assemblyai.py index f50f63380..f9db2c85e 100644 --- a/examples/transcription/transcription-assemblyai.py +++ b/examples/transcription/transcription-assemblyai.py @@ -48,7 +48,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = AssemblyAISTTService( - api_key=os.getenv("ASSEMBLYAI_API_KEY"), + api_key=os.environ["ASSEMBLYAI_API_KEY"], settings=AssemblyAISTTService.Settings( model="u3-rt-pro", ), diff --git a/examples/transcription/transcription-azure.py b/examples/transcription/transcription-azure.py index 301c6effd..1a551f745 100644 --- a/examples/transcription/transcription-azure.py +++ b/examples/transcription/transcription-azure.py @@ -54,7 +54,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = AzureSTTService( - api_key=os.getenv("AZURE_SPEECH_API_KEY"), + api_key=os.environ["AZURE_SPEECH_API_KEY"], region=os.getenv("AZURE_SPEECH_REGION"), ) diff --git a/examples/transcription/transcription-cartesia.py b/examples/transcription/transcription-cartesia.py index d3b83abb0..f86e5fa37 100644 --- a/examples/transcription/transcription-cartesia.py +++ b/examples/transcription/transcription-cartesia.py @@ -47,7 +47,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = CartesiaSTTService(api_key=os.getenv("CARTESIA_API_KEY")) + stt = CartesiaSTTService(api_key=os.environ["CARTESIA_API_KEY"]) tl = TranscriptionLogger() diff --git a/examples/transcription/transcription-deepgram.py b/examples/transcription/transcription-deepgram.py index 04246d237..92fe076e3 100644 --- a/examples/transcription/transcription-deepgram.py +++ b/examples/transcription/transcription-deepgram.py @@ -48,7 +48,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = DeepgramSTTService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], settings=DeepgramSTTService.Settings( language=Language.EN, ), diff --git a/examples/transcription/transcription-elevenlabs.py b/examples/transcription/transcription-elevenlabs.py index dbfd32ec6..548a12de2 100644 --- a/examples/transcription/transcription-elevenlabs.py +++ b/examples/transcription/transcription-elevenlabs.py @@ -49,7 +49,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = ElevenLabsRealtimeSTTService(api_key=os.getenv("ELEVENLABS_API_KEY")) + stt = ElevenLabsRealtimeSTTService(api_key=os.environ["ELEVENLABS_API_KEY"]) tl = TranscriptionLogger() vad_processor = VADProcessor(vad_analyzer=SileroVADAnalyzer()) diff --git a/examples/transcription/transcription-gladia-translation.py b/examples/transcription/transcription-gladia-translation.py index e6557cd15..477c73b23 100644 --- a/examples/transcription/transcription-gladia-translation.py +++ b/examples/transcription/transcription-gladia-translation.py @@ -55,9 +55,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") + region = os.getenv("GLADIA_REGION", "us-west") + assert region in ("us-west", "eu-west"), f"Invalid GLADIA_REGION: {region}" + stt = GladiaSTTService( - api_key=os.getenv("GLADIA_API_KEY"), - region=os.getenv("GLADIA_REGION"), + api_key=os.environ["GLADIA_API_KEY"], + region=region, settings=GladiaSTTService.Settings( language_config=LanguageConfig( languages=[Language.EN], diff --git a/examples/transcription/transcription-gladia.py b/examples/transcription/transcription-gladia.py index e98d4caf0..cd1152972 100644 --- a/examples/transcription/transcription-gladia.py +++ b/examples/transcription/transcription-gladia.py @@ -47,9 +47,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") + region = os.getenv("GLADIA_REGION", "us-west") + assert region in ("us-west", "eu-west"), f"Invalid GLADIA_REGION: {region}" + stt = GladiaSTTService( - api_key=os.getenv("GLADIA_API_KEY"), - region=os.getenv("GLADIA_REGION"), + api_key=os.environ["GLADIA_API_KEY"], + region=region, # settings=GladiaSTTSettings( # language_config=LanguageConfig( # languages=[Language.FR], diff --git a/examples/transcription/transcription-google-llm.py b/examples/transcription/transcription-google-llm.py index c22ed85a1..909711ecd 100644 --- a/examples/transcription/transcription-google-llm.py +++ b/examples/transcription/transcription-google-llm.py @@ -289,7 +289,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -301,7 +301,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): model="gemini-2.5-flash", system_instruction=conversation_system_message, ), - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], # we can give the GoogleLLMService a system instruction to use directly # in the GenerativeModel constructor. Let's do that rather than put # our system message in the messages list. @@ -313,7 +313,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): model="gemini-2.5-flash", system_instruction=transcriber_system_message, ), - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], ) messages = [ diff --git a/examples/transcription/transcription-gradium.py b/examples/transcription/transcription-gradium.py index 59140466e..0b7a4412d 100644 --- a/examples/transcription/transcription-gradium.py +++ b/examples/transcription/transcription-gradium.py @@ -50,7 +50,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = GradiumSTTService( - api_key=os.getenv("GRADIUM_API_KEY"), + api_key=os.environ["GRADIUM_API_KEY"], api_endpoint_base_url="wss://us.api.gradium.ai/api/speech/asr", settings=GradiumSTTService.Settings( language=Language.EN, diff --git a/examples/transcription/transcription-mistral.py b/examples/transcription/transcription-mistral.py index b040b457c..fa471d2e6 100644 --- a/examples/transcription/transcription-mistral.py +++ b/examples/transcription/transcription-mistral.py @@ -54,7 +54,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = MistralSTTService( - api_key=os.getenv("MISTRAL_API_KEY"), + api_key=os.environ["MISTRAL_API_KEY"], ) tl = TranscriptionLogger() diff --git a/examples/transcription/transcription-openai.py b/examples/transcription/transcription-openai.py index cbbf0e13d..5f3072177 100644 --- a/examples/transcription/transcription-openai.py +++ b/examples/transcription/transcription-openai.py @@ -50,7 +50,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = OpenAIRealtimeSTTService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIRealtimeSTTService.Settings( model="gpt-4o-transcribe", prompt="Expect words related to dogs, such as breed names.", diff --git a/examples/transcription/transcription-soniox.py b/examples/transcription/transcription-soniox.py index 9476e9441..b912fa04c 100644 --- a/examples/transcription/transcription-soniox.py +++ b/examples/transcription/transcription-soniox.py @@ -54,7 +54,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = SonioxSTTService( - api_key=os.getenv("SONIOX_API_KEY"), + api_key=os.environ["SONIOX_API_KEY"], ) tl = TranscriptionLogger() diff --git a/examples/transcription/transcription-speechmatics.py b/examples/transcription/transcription-speechmatics.py index 2df30f67b..2134c8c97 100644 --- a/examples/transcription/transcription-speechmatics.py +++ b/examples/transcription/transcription-speechmatics.py @@ -64,7 +64,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = SpeechmaticsSTTService( - api_key=os.getenv("SPEECHMATICS_API_KEY"), + api_key=os.environ["SPEECHMATICS_API_KEY"], settings=SpeechmaticsSTTService.Settings( language=Language.EN, speaker_active_format="<{speaker_id}>{text}", diff --git a/examples/transports/transports-daily.py b/examples/transports/transports-daily.py index c6695e2f8..09deda2dd 100644 --- a/examples/transports/transports-daily.py +++ b/examples/transports/transports-daily.py @@ -49,14 +49,14 @@ async def main(): ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/transports/transports-livekit.py b/examples/transports/transports-livekit.py index b2ec6b49c..8bb147c04 100644 --- a/examples/transports/transports-livekit.py +++ b/examples/transports/transports-livekit.py @@ -53,17 +53,17 @@ async def main(): ), ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), diff --git a/examples/transports/transports-small-webrtc.py b/examples/transports/transports-small-webrtc.py index 18403940b..2cb6d4bb0 100644 --- a/examples/transports/transports-small-webrtc.py +++ b/examples/transports/transports-small-webrtc.py @@ -62,17 +62,17 @@ async def run_example(webrtc_connection: SmallWebRTCConnection): ), ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/turn-management/turn-management-detect-user-idle.py b/examples/turn-management/turn-management-detect-user-idle.py index a159f0c4e..8391e3908 100644 --- a/examples/turn-management/turn-management-detect-user-idle.py +++ b/examples/turn-management/turn-management-detect-user-idle.py @@ -11,6 +11,7 @@ import os from dotenv import load_dotenv from loguru import logger +from pipecat.adapters.base_llm_adapter import LLMContextMessage from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -59,7 +60,7 @@ class IdleHandler: if self._retry_count == 1: # First attempt: Add a gentle prompt to the conversation - message = { + message: LLMContextMessage = { "role": "developer", "content": "The user has been quiet. Politely and briefly ask if they're still there.", } @@ -111,17 +112,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/turn-management/turn-management-filter-incomplete-turns.py b/examples/turn-management/turn-management-filter-incomplete-turns.py index fca89f66c..81273aaaa 100644 --- a/examples/turn-management/turn-management-filter-incomplete-turns.py +++ b/examples/turn-management/turn-management-filter-incomplete-turns.py @@ -66,17 +66,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), diff --git a/examples/turn-management/turn-management-interruption-config.py b/examples/turn-management/turn-management-interruption-config.py index d625910df..ee19075df 100644 --- a/examples/turn-management/turn-management-interruption-config.py +++ b/examples/turn-management/turn-management-interruption-config.py @@ -55,17 +55,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/turn-management/turn-management-smart-turn-local-coreml.py b/examples/turn-management/turn-management-smart-turn-local-coreml.py index 3042ceb5a..68e831bba 100644 --- a/examples/turn-management/turn-management-smart-turn-local-coreml.py +++ b/examples/turn-management/turn-management-smart-turn-local-coreml.py @@ -49,7 +49,7 @@ load_dotenv(override=True) # Then set the env variable: # export LOCAL_SMART_TURN_MODEL_PATH=./smart-turn # or add it to your .env file -smart_turn_model_path = os.getenv("LOCAL_SMART_TURN_MODEL_PATH") +smart_turn_model_path = os.environ["LOCAL_SMART_TURN_MODEL_PATH"] # We use lambdas to defer transport parameter creation until the transport # type is selected at runtime. @@ -72,17 +72,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/turn-management/turn-management-smart-turn-local.py b/examples/turn-management/turn-management-smart-turn-local.py index 7a73a4324..5509f0264 100644 --- a/examples/turn-management/turn-management-smart-turn-local.py +++ b/examples/turn-management/turn-management-smart-turn-local.py @@ -54,17 +54,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/turn-management/turn-management-turn-tracking-observer.py b/examples/turn-management/turn-management-turn-tracking-observer.py index 14d63d40b..86f061906 100644 --- a/examples/turn-management/turn-management-turn-tracking-observer.py +++ b/examples/turn-management/turn-management-turn-tracking-observer.py @@ -69,17 +69,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/turn-management/turn-management-user-assistant-turns.py b/examples/turn-management/turn-management-user-assistant-turns.py index 380c4866d..879eb1cdc 100644 --- a/examples/turn-management/turn-management-user-assistant-turns.py +++ b/examples/turn-management/turn-management-user-assistant-turns.py @@ -118,17 +118,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/turn-management/turn-management-user-mute-strategy.py b/examples/turn-management/turn-management-user-mute-strategy.py index 6e826f81d..0f9b24a8c 100644 --- a/examples/turn-management/turn-management-user-mute-strategy.py +++ b/examples/turn-management/turn-management-user-mute-strategy.py @@ -69,17 +69,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = DeepgramTTSService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], settings=DeepgramTTSService.Settings( voice="aura-2-helena-en", ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant who can check the weather. Always check the weather when a location is mentioned. Respond concisely and naturally. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points.", ), diff --git a/examples/update-settings/llm/llm-anthropic.py b/examples/update-settings/llm/llm-anthropic.py index 67ba6e9e9..885981ddd 100644 --- a/examples/update-settings/llm/llm-anthropic.py +++ b/examples/update-settings/llm/llm-anthropic.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), + api_key=os.environ["ANTHROPIC_API_KEY"], settings=AnthropicLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-aws-bedrock.py b/examples/update-settings/llm/llm-aws-bedrock.py index 168bb6bf0..6facff628 100644 --- a/examples/update-settings/llm/llm-aws-bedrock.py +++ b/examples/update-settings/llm/llm-aws-bedrock.py @@ -50,10 +50,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), diff --git a/examples/update-settings/llm/llm-aws-nova-sonic.py b/examples/update-settings/llm/llm-aws-nova-sonic.py index 99fa5cd7f..19ca3c805 100644 --- a/examples/update-settings/llm/llm-aws-nova-sonic.py +++ b/examples/update-settings/llm/llm-aws-nova-sonic.py @@ -49,9 +49,9 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") llm = AWSNovaSonicLLMService( - secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), - access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), - region=os.getenv("AWS_REGION"), + secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"], + access_key_id=os.environ["AWS_ACCESS_KEY_ID"], + region=os.environ["AWS_REGION"], settings=AWSNovaSonicLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-azure-realtime.py b/examples/update-settings/llm/llm-azure-realtime.py index fde633912..3d724bd53 100644 --- a/examples/update-settings/llm/llm-azure-realtime.py +++ b/examples/update-settings/llm/llm-azure-realtime.py @@ -10,6 +10,7 @@ import os from dotenv import load_dotenv from loguru import logger +from pipecat.adapters.base_llm_adapter import LLMContextMessage from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame from pipecat.pipeline.pipeline import Pipeline @@ -51,11 +52,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") llm = AzureRealtimeLLMService( - api_key=os.getenv("AZURE_REALTIME_API_KEY"), - base_url=os.getenv("AZURE_REALTIME_BASE_URL"), + api_key=os.environ["AZURE_REALTIME_API_KEY"], + base_url=os.environ["AZURE_REALTIME_BASE_URL"], ) - messages = [ + messages: list[LLMContextMessage] = [ { "role": "system", "content": "You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/update-settings/llm/llm-azure.py b/examples/update-settings/llm/llm-azure.py index 60d151484..3f9f5da57 100644 --- a/examples/update-settings/llm/llm-azure.py +++ b/examples/update-settings/llm/llm-azure.py @@ -50,18 +50,18 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = AzureLLMService( - api_key=os.getenv("AZURE_CHATGPT_API_KEY"), - endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + api_key=os.environ["AZURE_CHATGPT_API_KEY"], + endpoint=os.environ["AZURE_CHATGPT_ENDPOINT"], settings=AzureLLMService.Settings( model=os.getenv("AZURE_CHATGPT_MODEL"), system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/update-settings/llm/llm-cerebras.py b/examples/update-settings/llm/llm-cerebras.py index edda71d49..232723a3e 100644 --- a/examples/update-settings/llm/llm-cerebras.py +++ b/examples/update-settings/llm/llm-cerebras.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = CerebrasLLMService( - api_key=os.getenv("CEREBRAS_API_KEY"), + api_key=os.environ["CEREBRAS_API_KEY"], settings=CerebrasLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-deepseek.py b/examples/update-settings/llm/llm-deepseek.py index adcb63259..a7df0469b 100644 --- a/examples/update-settings/llm/llm-deepseek.py +++ b/examples/update-settings/llm/llm-deepseek.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = DeepSeekLLMService( - api_key=os.getenv("DEEPSEEK_API_KEY"), + api_key=os.environ["DEEPSEEK_API_KEY"], settings=DeepSeekLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-fireworks.py b/examples/update-settings/llm/llm-fireworks.py index 3468d8dec..154bb63ce 100644 --- a/examples/update-settings/llm/llm-fireworks.py +++ b/examples/update-settings/llm/llm-fireworks.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = FireworksLLMService( - api_key=os.getenv("FIREWORKS_API_KEY"), + api_key=os.environ["FIREWORKS_API_KEY"], settings=FireworksLLMService.Settings( model="accounts/fireworks/models/gpt-oss-20b", system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/update-settings/llm/llm-gemini-live-vertex.py b/examples/update-settings/llm/llm-gemini-live-vertex.py index c18fb34c5..3992be309 100644 --- a/examples/update-settings/llm/llm-gemini-live-vertex.py +++ b/examples/update-settings/llm/llm-gemini-live-vertex.py @@ -49,9 +49,9 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") llm = GeminiLiveVertexLLMService( - credentials=os.getenv("GOOGLE_VERTEX_TEST_CREDENTIALS"), - project_id=os.getenv("GOOGLE_CLOUD_PROJECT_ID"), - location=os.getenv("GOOGLE_CLOUD_LOCATION"), + credentials=os.environ["GOOGLE_VERTEX_TEST_CREDENTIALS"], + project_id=os.environ["GOOGLE_CLOUD_PROJECT_ID"], + location=os.environ["GOOGLE_CLOUD_LOCATION"], settings=GeminiLiveVertexLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-gemini-live.py b/examples/update-settings/llm/llm-gemini-live.py index dafc9efe4..0c066648e 100644 --- a/examples/update-settings/llm/llm-gemini-live.py +++ b/examples/update-settings/llm/llm-gemini-live.py @@ -49,7 +49,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") llm = GeminiLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GeminiLiveLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-google-vertex.py b/examples/update-settings/llm/llm-google-vertex.py index b63b176bc..7c4ca8446 100644 --- a/examples/update-settings/llm/llm-google-vertex.py +++ b/examples/update-settings/llm/llm-google-vertex.py @@ -50,10 +50,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -61,8 +61,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = GoogleVertexLLMService( credentials=os.getenv("GOOGLE_VERTEX_TEST_CREDENTIALS"), - project_id=os.getenv("GOOGLE_CLOUD_PROJECT_ID"), - location=os.getenv("GOOGLE_CLOUD_LOCATION"), + project_id=os.environ["GOOGLE_CLOUD_PROJECT_ID"], + location=os.environ["GOOGLE_CLOUD_LOCATION"], settings=GoogleVertexLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-google.py b/examples/update-settings/llm/llm-google.py index 09a9d686d..69b474190 100644 --- a/examples/update-settings/llm/llm-google.py +++ b/examples/update-settings/llm/llm-google.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-grok-realtime.py b/examples/update-settings/llm/llm-grok-realtime.py index 304ccce8e..4ab28cdda 100644 --- a/examples/update-settings/llm/llm-grok-realtime.py +++ b/examples/update-settings/llm/llm-grok-realtime.py @@ -10,6 +10,7 @@ import os from dotenv import load_dotenv from loguru import logger +from pipecat.adapters.base_llm_adapter import LLMContextMessage from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame from pipecat.pipeline.pipeline import Pipeline @@ -50,9 +51,9 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - llm = GrokRealtimeLLMService(api_key=os.getenv("XAI_API_KEY")) + llm = GrokRealtimeLLMService(api_key=os.environ["XAI_API_KEY"]) - messages = [ + messages: list[LLMContextMessage] = [ { "role": "system", "content": "You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/update-settings/llm/llm-grok.py b/examples/update-settings/llm/llm-grok.py index dedb7b950..463b27807 100644 --- a/examples/update-settings/llm/llm-grok.py +++ b/examples/update-settings/llm/llm-grok.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = GrokLLMService( - api_key=os.getenv("XAI_API_KEY"), + api_key=os.environ["XAI_API_KEY"], settings=GrokLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-groq.py b/examples/update-settings/llm/llm-groq.py index 771a5cbe9..365d8b4d4 100644 --- a/examples/update-settings/llm/llm-groq.py +++ b/examples/update-settings/llm/llm-groq.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = GroqLLMService( - api_key=os.getenv("GROQ_API_KEY"), + api_key=os.environ["GROQ_API_KEY"], settings=GroqLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-mistral.py b/examples/update-settings/llm/llm-mistral.py index 5da4d47b7..4aa66197e 100644 --- a/examples/update-settings/llm/llm-mistral.py +++ b/examples/update-settings/llm/llm-mistral.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = MistralLLMService( - api_key=os.getenv("MISTRAL_API_KEY"), + api_key=os.environ["MISTRAL_API_KEY"], settings=MistralLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-nvidia.py b/examples/update-settings/llm/llm-nvidia.py index 06c45dd5d..a44083821 100644 --- a/examples/update-settings/llm/llm-nvidia.py +++ b/examples/update-settings/llm/llm-nvidia.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = NvidiaLLMService( - api_key=os.getenv("NVIDIA_API_KEY"), + api_key=os.environ["NVIDIA_API_KEY"], settings=NvidiaLLMService.Settings( model="meta/llama-3.1-405b-instruct", system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/update-settings/llm/llm-ollama.py b/examples/update-settings/llm/llm-ollama.py index 90ddcbfb0..ac9892bee 100644 --- a/examples/update-settings/llm/llm-ollama.py +++ b/examples/update-settings/llm/llm-ollama.py @@ -50,10 +50,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), diff --git a/examples/update-settings/llm/llm-openai-realtime.py b/examples/update-settings/llm/llm-openai-realtime.py index 83cdb9fa7..120d8a86e 100644 --- a/examples/update-settings/llm/llm-openai-realtime.py +++ b/examples/update-settings/llm/llm-openai-realtime.py @@ -10,6 +10,7 @@ import os from dotenv import load_dotenv from loguru import logger +from pipecat.adapters.base_llm_adapter import LLMContextMessage from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame from pipecat.pipeline.pipeline import Pipeline @@ -50,9 +51,9 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - llm = OpenAIRealtimeLLMService(api_key=os.getenv("OPENAI_API_KEY")) + llm = OpenAIRealtimeLLMService(api_key=os.environ["OPENAI_API_KEY"]) - messages = [ + messages: list[LLMContextMessage] = [ { "role": "system", "content": "You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/update-settings/llm/llm-openai-responses-http.py b/examples/update-settings/llm/llm-openai-responses-http.py index 051a2f71e..0e7d6ec44 100644 --- a/examples/update-settings/llm/llm-openai-responses-http.py +++ b/examples/update-settings/llm/llm-openai-responses-http.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesHttpLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIResponsesHttpLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-openai-responses.py b/examples/update-settings/llm/llm-openai-responses.py index 8c214b639..d4857bab1 100644 --- a/examples/update-settings/llm/llm-openai-responses.py +++ b/examples/update-settings/llm/llm-openai-responses.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIResponsesLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-openai.py b/examples/update-settings/llm/llm-openai.py index 15ee4554a..dee6ba8cf 100644 --- a/examples/update-settings/llm/llm-openai.py +++ b/examples/update-settings/llm/llm-openai.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-openrouter.py b/examples/update-settings/llm/llm-openrouter.py index 7b8aef6d8..3c554b921 100644 --- a/examples/update-settings/llm/llm-openrouter.py +++ b/examples/update-settings/llm/llm-openrouter.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenRouterLLMService( - api_key=os.getenv("OPENROUTER_API_KEY"), + api_key=os.environ["OPENROUTER_API_KEY"], settings=OpenRouterLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-perplexity.py b/examples/update-settings/llm/llm-perplexity.py index 99a40c614..d1e2aae4a 100644 --- a/examples/update-settings/llm/llm-perplexity.py +++ b/examples/update-settings/llm/llm-perplexity.py @@ -50,25 +50,18 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) - llm = PerplexityLLMService(api_key=os.getenv("PERPLEXITY_API_KEY")) + llm = PerplexityLLMService(api_key=os.environ["PERPLEXITY_API_KEY"]) - messages = [ - { - "role": "developer", - "content": "You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. Start by introducing yourself.", - }, - ] - - context = LLMContext(messages) + context = LLMContext() user_aggregator, assistant_aggregator = LLMContextAggregatorPair( context, user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), @@ -98,6 +91,9 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info(f"Client connected") + context.add_message( + {"role": "developer", "content": "Please introduce yourself to the user."} + ) await task.queue_frames([LLMRunFrame()]) await asyncio.sleep(10) diff --git a/examples/update-settings/llm/llm-qwen.py b/examples/update-settings/llm/llm-qwen.py index ff7d126d3..585a7c2da 100644 --- a/examples/update-settings/llm/llm-qwen.py +++ b/examples/update-settings/llm/llm-qwen.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = QwenLLMService( - api_key=os.getenv("QWEN_API_KEY"), + api_key=os.environ["QWEN_API_KEY"], settings=QwenLLMService.Settings( model="qwen2.5-72b-instruct", system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/update-settings/llm/llm-sambanova.py b/examples/update-settings/llm/llm-sambanova.py index 4f03db5f2..c543fcee2 100644 --- a/examples/update-settings/llm/llm-sambanova.py +++ b/examples/update-settings/llm/llm-sambanova.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = SambaNovaLLMService( - api_key=os.getenv("SAMBANOVA_API_KEY"), + api_key=os.environ["SAMBANOVA_API_KEY"], settings=SambaNovaLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-sarvam.py b/examples/update-settings/llm/llm-sarvam.py index 62102238f..2966f46b8 100644 --- a/examples/update-settings/llm/llm-sarvam.py +++ b/examples/update-settings/llm/llm-sarvam.py @@ -47,31 +47,26 @@ transport_params = { } -def _require_env(name: str) -> str: - value = os.getenv(name) - if not value: - raise ValueError(f"Environment variable `{name}` is required.") - return value - - async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info("Starting bot") stt = SarvamSTTService( settings=SarvamSTTService.Settings(model="saaras:v3"), - api_key=_require_env("SARVAM_API_KEY"), + api_key=os.environ["SARVAM_API_KEY"], ) tts = SarvamTTSService( settings=SarvamTTSService.Settings(model="bulbul:v3"), - api_key=_require_env("SARVAM_API_KEY"), + api_key=os.environ["SARVAM_API_KEY"], ) llm = SarvamLLMService( - api_key=_require_env("SARVAM_API_KEY"), - settings=SarvamLLMService.Settings(model="sarvam-30b"), - system_instruction=( - "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way." + api_key=os.environ["SARVAM_API_KEY"], + settings=SarvamLLMService.Settings( + model="sarvam-30b", + system_instruction=( + "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way." + ), ), ) diff --git a/examples/update-settings/llm/llm-together.py b/examples/update-settings/llm/llm-together.py index 3144f1906..2e1a8cb7a 100644 --- a/examples/update-settings/llm/llm-together.py +++ b/examples/update-settings/llm/llm-together.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = TogetherLLMService( - api_key=os.getenv("TOGETHER_API_KEY"), + api_key=os.environ["TOGETHER_API_KEY"], settings=TogetherLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-ultravox-realtime.py b/examples/update-settings/llm/llm-ultravox-realtime.py index f77aa4252..4c530735e 100644 --- a/examples/update-settings/llm/llm-ultravox-realtime.py +++ b/examples/update-settings/llm/llm-ultravox-realtime.py @@ -11,6 +11,7 @@ import os from dotenv import load_dotenv from loguru import logger +from pipecat.adapters.base_llm_adapter import LLMContextMessage from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame @@ -55,7 +56,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = UltravoxRealtimeLLMService( params=OneShotInputParams( - api_key=os.getenv("ULTRAVOX_API_KEY"), + api_key=os.environ["ULTRAVOX_API_KEY"], system_prompt=system_prompt, temperature=0.3, max_duration=datetime.timedelta(minutes=3), @@ -63,7 +64,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): one_shot_selected_tools=ToolsSchema(standard_tools=[]), ) - messages = [ + messages: list[LLMContextMessage] = [ { "role": "system", "content": system_prompt, diff --git a/examples/update-settings/stt/stt-assemblyai.py b/examples/update-settings/stt/stt-assemblyai.py index 8017b6021..276499bb1 100644 --- a/examples/update-settings/stt/stt-assemblyai.py +++ b/examples/update-settings/stt/stt-assemblyai.py @@ -51,21 +51,21 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = AssemblyAISTTService( - api_key=os.getenv("ASSEMBLYAI_API_KEY"), + api_key=os.environ["ASSEMBLYAI_API_KEY"], settings=AssemblyAISTTService.Settings( model="u3-rt-pro", ), ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Try saying difficult names like 'Xiomara', 'Saoirse', or 'Krzystof' to test transcription accuracy.", ), diff --git a/examples/update-settings/stt/stt-aws-transcribe.py b/examples/update-settings/stt/stt-aws-transcribe.py index 25c2f272a..503ecae39 100644 --- a/examples/update-settings/stt/stt-aws-transcribe.py +++ b/examples/update-settings/stt/stt-aws-transcribe.py @@ -54,14 +54,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): stt = AWSTranscribeSTTService() tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-azure.py b/examples/update-settings/stt/stt-azure.py index adc242d3c..e1f8b62ad 100644 --- a/examples/update-settings/stt/stt-azure.py +++ b/examples/update-settings/stt/stt-azure.py @@ -52,19 +52,19 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = AzureSTTService( - api_key=os.getenv("AZURE_SPEECH_API_KEY"), + api_key=os.environ["AZURE_SPEECH_API_KEY"], region=os.getenv("AZURE_SPEECH_REGION"), ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-cartesia.py b/examples/update-settings/stt/stt-cartesia.py index 79daec9fd..2f5602b8d 100644 --- a/examples/update-settings/stt/stt-cartesia.py +++ b/examples/update-settings/stt/stt-cartesia.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = CartesiaSTTService(api_key=os.getenv("CARTESIA_API_KEY")) + stt = CartesiaSTTService(api_key=os.environ["CARTESIA_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-deepgram-flux.py b/examples/update-settings/stt/stt-deepgram-flux.py index 5db99a327..f8810e2fd 100644 --- a/examples/update-settings/stt/stt-deepgram-flux.py +++ b/examples/update-settings/stt/stt-deepgram-flux.py @@ -56,7 +56,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # across English and Spanish. TranscriptionFrame.language will reflect # whichever language Flux detected for each turn. stt = DeepgramFluxSTTService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], settings=DeepgramFluxSTTService.Settings( model="flux-general-multi", language_hints=[Language.EN, Language.ES], @@ -64,14 +64,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-deepgram-sagemaker.py b/examples/update-settings/stt/stt-deepgram-sagemaker.py index aa61eaa1e..05a655abc 100644 --- a/examples/update-settings/stt/stt-deepgram-sagemaker.py +++ b/examples/update-settings/stt/stt-deepgram-sagemaker.py @@ -52,19 +52,19 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = DeepgramSageMakerSTTService( - endpoint_name=os.getenv("SAGEMAKER_STT_ENDPOINT_NAME"), - region=os.getenv("AWS_REGION"), + endpoint_name=os.environ["SAGEMAKER_STT_ENDPOINT_NAME"], + region=os.environ["AWS_REGION"], ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-deepgram.py b/examples/update-settings/stt/stt-deepgram.py index a083244d0..9f345b1c3 100644 --- a/examples/update-settings/stt/stt-deepgram.py +++ b/examples/update-settings/stt/stt-deepgram.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-elevenlabs-realtime.py b/examples/update-settings/stt/stt-elevenlabs-realtime.py index f2cba36ec..8a195ad48 100644 --- a/examples/update-settings/stt/stt-elevenlabs-realtime.py +++ b/examples/update-settings/stt/stt-elevenlabs-realtime.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = ElevenLabsRealtimeSTTService(api_key=os.getenv("ELEVENLABS_API_KEY")) + stt = ElevenLabsRealtimeSTTService(api_key=os.environ["ELEVENLABS_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-elevenlabs.py b/examples/update-settings/stt/stt-elevenlabs.py index 03efdc71e..3b3623264 100644 --- a/examples/update-settings/stt/stt-elevenlabs.py +++ b/examples/update-settings/stt/stt-elevenlabs.py @@ -54,19 +54,19 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): async with aiohttp.ClientSession() as session: stt = ElevenLabsSTTService( - api_key=os.getenv("ELEVENLABS_API_KEY"), + api_key=os.environ["ELEVENLABS_API_KEY"], aiohttp_session=session, ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-fal.py b/examples/update-settings/stt/stt-fal.py index 83b7dbc79..5dccd4d36 100644 --- a/examples/update-settings/stt/stt-fal.py +++ b/examples/update-settings/stt/stt-fal.py @@ -53,14 +53,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): stt = FalSTTService(api_key=os.getenv("FAL_KEY")) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-gladia.py b/examples/update-settings/stt/stt-gladia.py index 972456853..4449eff06 100644 --- a/examples/update-settings/stt/stt-gladia.py +++ b/examples/update-settings/stt/stt-gladia.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = GladiaSTTService(api_key=os.getenv("GLADIA_API_KEY")) + stt = GladiaSTTService(api_key=os.environ["GLADIA_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-google.py b/examples/update-settings/stt/stt-google.py index 58b0ff357..76ea1325d 100644 --- a/examples/update-settings/stt/stt-google.py +++ b/examples/update-settings/stt/stt-google.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = GoogleSTTService(credentials=os.getenv("GOOGLE_TEST_CREDENTIALS")) + stt = GoogleSTTService(credentials=os.environ["GOOGLE_TEST_CREDENTIALS"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-gradium.py b/examples/update-settings/stt/stt-gradium.py index aebced030..876b20260 100644 --- a/examples/update-settings/stt/stt-gradium.py +++ b/examples/update-settings/stt/stt-gradium.py @@ -51,19 +51,19 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = GradiumSTTService( - api_key=os.getenv("GRADIUM_API_KEY"), + api_key=os.environ["GRADIUM_API_KEY"], api_endpoint_base_url="wss://us.api.gradium.ai/api/speech/asr", ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-groq.py b/examples/update-settings/stt/stt-groq.py index f612ba05b..033ed4c03 100644 --- a/examples/update-settings/stt/stt-groq.py +++ b/examples/update-settings/stt/stt-groq.py @@ -51,18 +51,18 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = GroqSTTService( - api_key=os.getenv("GROQ_API_KEY"), + api_key=os.environ["GROQ_API_KEY"], ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-nvidia-segmented.py b/examples/update-settings/stt/stt-nvidia-segmented.py index d2880d446..1cce6fba2 100644 --- a/examples/update-settings/stt/stt-nvidia-segmented.py +++ b/examples/update-settings/stt/stt-nvidia-segmented.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = NvidiaSegmentedSTTService(api_key=os.getenv("NVIDIA_API_KEY")) + stt = NvidiaSegmentedSTTService(api_key=os.environ["NVIDIA_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-nvidia.py b/examples/update-settings/stt/stt-nvidia.py index e339218e8..499269f86 100644 --- a/examples/update-settings/stt/stt-nvidia.py +++ b/examples/update-settings/stt/stt-nvidia.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = NvidiaSTTService(api_key=os.getenv("NVIDIA_API_KEY")) + stt = NvidiaSTTService(api_key=os.environ["NVIDIA_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-openai-realtime.py b/examples/update-settings/stt/stt-openai-realtime.py index e5366eb7f..33c4c4e7d 100644 --- a/examples/update-settings/stt/stt-openai-realtime.py +++ b/examples/update-settings/stt/stt-openai-realtime.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = OpenAIRealtimeSTTService(api_key=os.getenv("OPENAI_API_KEY")) + stt = OpenAIRealtimeSTTService(api_key=os.environ["OPENAI_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-sarvam.py b/examples/update-settings/stt/stt-sarvam.py index ddb76275d..60da61359 100644 --- a/examples/update-settings/stt/stt-sarvam.py +++ b/examples/update-settings/stt/stt-sarvam.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = SarvamSTTService(api_key=os.getenv("SARVAM_API_KEY")) + stt = SarvamSTTService(api_key=os.environ["SARVAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-soniox.py b/examples/update-settings/stt/stt-soniox.py index 05b3c267f..adf3fb547 100644 --- a/examples/update-settings/stt/stt-soniox.py +++ b/examples/update-settings/stt/stt-soniox.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = SonioxSTTService(api_key=os.getenv("SONIOX_API_KEY")) + stt = SonioxSTTService(api_key=os.environ["SONIOX_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-speechmatics.py b/examples/update-settings/stt/stt-speechmatics.py index 93c97c827..f92d2ff7e 100644 --- a/examples/update-settings/stt/stt-speechmatics.py +++ b/examples/update-settings/stt/stt-speechmatics.py @@ -52,7 +52,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = SpeechmaticsSTTService( - api_key=os.getenv("SPEECHMATICS_API_KEY"), + api_key=os.environ["SPEECHMATICS_API_KEY"], settings=SpeechmaticsSTTService.Settings( enable_diarization=True, speaker_active_format="<{speaker_id}>{text}", @@ -61,14 +61,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-whisper-api.py b/examples/update-settings/stt/stt-whisper-api.py index 84dea00a2..372e6a36e 100644 --- a/examples/update-settings/stt/stt-whisper-api.py +++ b/examples/update-settings/stt/stt-whisper-api.py @@ -56,18 +56,18 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # - SambaNova # - Groq stt = OpenAISTTService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-whisper-mlx.py b/examples/update-settings/stt/stt-whisper-mlx.py index a13c053cf..7e9e5e4e1 100644 --- a/examples/update-settings/stt/stt-whisper-mlx.py +++ b/examples/update-settings/stt/stt-whisper-mlx.py @@ -57,14 +57,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-whisper.py b/examples/update-settings/stt/stt-whisper.py index d09c4f3b4..e25f3915b 100644 --- a/examples/update-settings/stt/stt-whisper.py +++ b/examples/update-settings/stt/stt-whisper.py @@ -57,14 +57,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-asyncai-http.py b/examples/update-settings/tts/tts-asyncai-http.py index 93ca099b0..5f6342114 100644 --- a/examples/update-settings/tts/tts-asyncai-http.py +++ b/examples/update-settings/tts/tts-asyncai-http.py @@ -55,7 +55,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = AsyncAIHttpTTSService( api_key=os.getenv("ASYNCAI_API_KEY", ""), @@ -66,7 +66,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-asyncai.py b/examples/update-settings/tts/tts-asyncai.py index a24e37c9e..3d7701288 100644 --- a/examples/update-settings/tts/tts-asyncai.py +++ b/examples/update-settings/tts/tts-asyncai.py @@ -51,7 +51,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = AsyncAITTSService( api_key=os.getenv("ASYNCAI_API_KEY", ""), @@ -61,7 +61,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-aws-polly.py b/examples/update-settings/tts/tts-aws-polly.py index b00c6982b..6ddeaf04b 100644 --- a/examples/update-settings/tts/tts-aws-polly.py +++ b/examples/update-settings/tts/tts-aws-polly.py @@ -50,12 +50,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = AWSPollyTTSService() llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-azure-http.py b/examples/update-settings/tts/tts-azure-http.py index a0193b063..5b3d6adb3 100644 --- a/examples/update-settings/tts/tts-azure-http.py +++ b/examples/update-settings/tts/tts-azure-http.py @@ -50,15 +50,15 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = AzureHttpTTSService( - api_key=os.getenv("AZURE_SPEECH_API_KEY"), - region=os.getenv("AZURE_SPEECH_REGION"), + api_key=os.environ["AZURE_SPEECH_API_KEY"], + region=os.environ["AZURE_SPEECH_REGION"], ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-azure.py b/examples/update-settings/tts/tts-azure.py index 46daa6781..a3a1eb822 100644 --- a/examples/update-settings/tts/tts-azure.py +++ b/examples/update-settings/tts/tts-azure.py @@ -50,15 +50,15 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = AzureTTSService( - api_key=os.getenv("AZURE_SPEECH_API_KEY"), - region=os.getenv("AZURE_SPEECH_REGION"), + api_key=os.environ["AZURE_SPEECH_API_KEY"], + region=os.environ["AZURE_SPEECH_REGION"], ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-camb.py b/examples/update-settings/tts/tts-camb.py index 97166ad0f..874489d97 100644 --- a/examples/update-settings/tts/tts-camb.py +++ b/examples/update-settings/tts/tts-camb.py @@ -51,12 +51,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) - tts = CambTTSService(api_key=os.getenv("CAMB_API_KEY")) + tts = CambTTSService(api_key=os.environ["CAMB_API_KEY"]) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-cartesia-http.py b/examples/update-settings/tts/tts-cartesia-http.py index 84d3188af..b126d1dee 100644 --- a/examples/update-settings/tts/tts-cartesia-http.py +++ b/examples/update-settings/tts/tts-cartesia-http.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaHttpTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaHttpTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-cartesia.py b/examples/update-settings/tts/tts-cartesia.py index 71d804875..7f267a4d7 100644 --- a/examples/update-settings/tts/tts-cartesia.py +++ b/examples/update-settings/tts/tts-cartesia.py @@ -52,17 +52,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-deepgram-http.py b/examples/update-settings/tts/tts-deepgram-http.py index 16083e95e..0aa24c4e5 100644 --- a/examples/update-settings/tts/tts-deepgram-http.py +++ b/examples/update-settings/tts/tts-deepgram-http.py @@ -54,15 +54,15 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = DeepgramHttpTTSService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], aiohttp_session=session, ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-deepgram-sagemaker.py b/examples/update-settings/tts/tts-deepgram-sagemaker.py index 1dfd2a365..47329b1f3 100644 --- a/examples/update-settings/tts/tts-deepgram-sagemaker.py +++ b/examples/update-settings/tts/tts-deepgram-sagemaker.py @@ -50,16 +50,16 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = DeepgramSageMakerTTSService( - endpoint_name=os.getenv("SAGEMAKER_TTS_ENDPOINT_NAME"), - region=os.getenv("AWS_REGION"), + endpoint_name=os.environ["SAGEMAKER_TTS_ENDPOINT_NAME"], + region=os.environ["AWS_REGION"], voice="aura-2-helena-en", ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-deepgram.py b/examples/update-settings/tts/tts-deepgram.py index 596d12dfa..8e532937a 100644 --- a/examples/update-settings/tts/tts-deepgram.py +++ b/examples/update-settings/tts/tts-deepgram.py @@ -50,12 +50,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) - tts = DeepgramTTSService(api_key=os.getenv("DEEPGRAM_API_KEY")) + tts = DeepgramTTSService(api_key=os.environ["DEEPGRAM_API_KEY"]) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-elevenlabs-http.py b/examples/update-settings/tts/tts-elevenlabs-http.py index 60dab3017..ac85a1452 100644 --- a/examples/update-settings/tts/tts-elevenlabs-http.py +++ b/examples/update-settings/tts/tts-elevenlabs-http.py @@ -54,16 +54,21 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = ElevenLabsHttpTTSService( - api_key=os.getenv("ELEVENLABS_API_KEY"), - settings=ElevenLabsHttpTTSService.Settings(voice=os.getenv("ELEVENLABS_VOICE_ID")), + api_key=os.environ["ELEVENLABS_API_KEY"], + settings=ElevenLabsHttpTTSService.Settings( + voice=os.getenv( + "ELEVENLABS_VOICE_ID", + "Xb7hH8MSUJpSbSDYk0k2", + ) + ), aiohttp_session=session, ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-elevenlabs.py b/examples/update-settings/tts/tts-elevenlabs.py index 83de8165f..9aeadf993 100644 --- a/examples/update-settings/tts/tts-elevenlabs.py +++ b/examples/update-settings/tts/tts-elevenlabs.py @@ -50,15 +50,20 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = ElevenLabsTTSService( - api_key=os.getenv("ELEVENLABS_API_KEY"), - settings=ElevenLabsTTSService.Settings(voice=os.getenv("ELEVENLABS_VOICE_ID")), + api_key=os.environ["ELEVENLABS_API_KEY"], + settings=ElevenLabsTTSService.Settings( + voice=os.getenv( + "ELEVENLABS_VOICE_ID", + "Xb7hH8MSUJpSbSDYk0k2", + ) + ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), @@ -109,7 +114,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info("Updating ElevenLabs TTS settings: switching to a different voice") await task.queue_frame( TTSUpdateSettingsFrame( - delta=ElevenLabsTTSService.Settings(voice=os.getenv("ELEVENLABS_VOICE_ID_ALT")) + delta=ElevenLabsTTSService.Settings( + voice=os.getenv( + "ELEVENLABS_VOICE_ID_ALT", + "CwhRBWXzGAHq8TQ4Fs17", + ) + ) ) ) diff --git a/examples/update-settings/tts/tts-fish.py b/examples/update-settings/tts/tts-fish.py index f486ad9d1..44566d915 100644 --- a/examples/update-settings/tts/tts-fish.py +++ b/examples/update-settings/tts/tts-fish.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = FishAudioTTSService( - api_key=os.getenv("FISH_API_KEY"), + api_key=os.environ["FISH_API_KEY"], settings=FishAudioTTSService.Settings( voice="4ce7e917cedd4bc2bb2e6ff3a46acaa1" ), # Barack Obama ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-gemini.py b/examples/update-settings/tts/tts-gemini.py index e984c3575..a6b56ac54 100644 --- a/examples/update-settings/tts/tts-gemini.py +++ b/examples/update-settings/tts/tts-gemini.py @@ -51,10 +51,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = GeminiTTSService( - credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + credentials=os.environ["GOOGLE_TEST_CREDENTIALS"], settings=GeminiTTSService.Settings( model="gemini-2.5-flash-tts", voice="Charon", @@ -64,7 +64,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-google-http.py b/examples/update-settings/tts/tts-google-http.py index b9a605560..eef39d8f8 100644 --- a/examples/update-settings/tts/tts-google-http.py +++ b/examples/update-settings/tts/tts-google-http.py @@ -50,12 +50,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) - tts = GoogleHttpTTSService(credentials=os.getenv("GOOGLE_TEST_CREDENTIALS")) + tts = GoogleHttpTTSService(credentials=os.environ["GOOGLE_TEST_CREDENTIALS"]) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-google-stream.py b/examples/update-settings/tts/tts-google-stream.py index a1c5129a6..2614022e7 100644 --- a/examples/update-settings/tts/tts-google-stream.py +++ b/examples/update-settings/tts/tts-google-stream.py @@ -50,12 +50,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) - tts = GoogleTTSService(credentials=os.getenv("GOOGLE_TEST_CREDENTIALS")) + tts = GoogleTTSService(credentials=os.environ["GOOGLE_TEST_CREDENTIALS"]) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-gradium.py b/examples/update-settings/tts/tts-gradium.py index 7498557a9..3750b004a 100644 --- a/examples/update-settings/tts/tts-gradium.py +++ b/examples/update-settings/tts/tts-gradium.py @@ -50,16 +50,16 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = GradiumTTSService( - api_key=os.getenv("GRADIUM_API_KEY"), + api_key=os.environ["GRADIUM_API_KEY"], settings=GradiumTTSService.Settings(voice="YTpq7expH9539ERJ"), url="wss://us.api.gradium.ai/api/speech/tts", ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-groq.py b/examples/update-settings/tts/tts-groq.py index 757276c6a..1926dfa96 100644 --- a/examples/update-settings/tts/tts-groq.py +++ b/examples/update-settings/tts/tts-groq.py @@ -50,12 +50,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) - tts = GroqTTSService(api_key=os.getenv("GROQ_API_KEY")) + tts = GroqTTSService(api_key=os.environ["GROQ_API_KEY"]) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-hume.py b/examples/update-settings/tts/tts-hume.py index 034ff6797..6399a908a 100644 --- a/examples/update-settings/tts/tts-hume.py +++ b/examples/update-settings/tts/tts-hume.py @@ -50,7 +50,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = HumeTTSService( api_key=os.getenv("HUME_API_KEY"), @@ -58,7 +58,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-inworld-http.py b/examples/update-settings/tts/tts-inworld-http.py index 41b960dd4..28ea6c81d 100644 --- a/examples/update-settings/tts/tts-inworld-http.py +++ b/examples/update-settings/tts/tts-inworld-http.py @@ -54,12 +54,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) - tts = InworldHttpTTSService(api_key=os.getenv("INWORLD_API_KEY"), aiohttp_session=session) + tts = InworldHttpTTSService(api_key=os.environ["INWORLD_API_KEY"], aiohttp_session=session) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-inworld.py b/examples/update-settings/tts/tts-inworld.py index 9f40247dd..9bdd706e5 100644 --- a/examples/update-settings/tts/tts-inworld.py +++ b/examples/update-settings/tts/tts-inworld.py @@ -50,12 +50,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) - tts = InworldTTSService(api_key=os.getenv("INWORLD_API_KEY")) + tts = InworldTTSService(api_key=os.environ["INWORLD_API_KEY"]) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-kokoro.py b/examples/update-settings/tts/tts-kokoro.py index 8c7e1f492..053dd630b 100644 --- a/examples/update-settings/tts/tts-kokoro.py +++ b/examples/update-settings/tts/tts-kokoro.py @@ -50,7 +50,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = KokoroTTSService( settings=KokoroTTSService.Settings( @@ -59,7 +59,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-lmnt.py b/examples/update-settings/tts/tts-lmnt.py index d700e22c2..68f5f6d12 100644 --- a/examples/update-settings/tts/tts-lmnt.py +++ b/examples/update-settings/tts/tts-lmnt.py @@ -50,15 +50,15 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = LmntTTSService( - api_key=os.getenv("LMNT_API_KEY"), + api_key=os.environ["LMNT_API_KEY"], settings=LmntTTSService.Settings(voice="lily"), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-minimax.py b/examples/update-settings/tts/tts-minimax.py index b77e84554..3a02adce0 100644 --- a/examples/update-settings/tts/tts-minimax.py +++ b/examples/update-settings/tts/tts-minimax.py @@ -52,16 +52,16 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = MiniMaxHttpTTSService( - api_key=os.getenv("MINIMAX_API_KEY", ""), - group_id=os.getenv("MINIMAX_GROUP_ID", ""), + api_key=os.environ["MINIMAX_API_KEY"], + group_id=os.environ["MINIMAX_GROUP_ID"], aiohttp_session=session, ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-neuphonic-http.py b/examples/update-settings/tts/tts-neuphonic-http.py index 3db9e5d2e..a55d6fd6c 100644 --- a/examples/update-settings/tts/tts-neuphonic-http.py +++ b/examples/update-settings/tts/tts-neuphonic-http.py @@ -51,16 +51,16 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) async with aiohttp.ClientSession() as session: tts = NeuphonicHttpTTSService( - api_key=os.getenv("NEUPHONIC_API_KEY"), + api_key=os.environ["NEUPHONIC_API_KEY"], aiohttp_session=session, ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-neuphonic.py b/examples/update-settings/tts/tts-neuphonic.py index bb10f8699..fd37ec40e 100644 --- a/examples/update-settings/tts/tts-neuphonic.py +++ b/examples/update-settings/tts/tts-neuphonic.py @@ -50,12 +50,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) - tts = NeuphonicTTSService(api_key=os.getenv("NEUPHONIC_API_KEY")) + tts = NeuphonicTTSService(api_key=os.environ["NEUPHONIC_API_KEY"]) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-nvidia.py b/examples/update-settings/tts/tts-nvidia.py index 2b41e1cdd..e136db033 100644 --- a/examples/update-settings/tts/tts-nvidia.py +++ b/examples/update-settings/tts/tts-nvidia.py @@ -51,12 +51,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) - tts = NvidiaTTSService(api_key=os.getenv("NVIDIA_API_KEY")) + tts = NvidiaTTSService(api_key=os.environ["NVIDIA_API_KEY"]) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-openai.py b/examples/update-settings/tts/tts-openai.py index c75e56513..e23db32ac 100644 --- a/examples/update-settings/tts/tts-openai.py +++ b/examples/update-settings/tts/tts-openai.py @@ -50,12 +50,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) - tts = OpenAITTSService(api_key=os.getenv("OPENAI_API_KEY")) + tts = OpenAITTSService(api_key=os.environ["OPENAI_API_KEY"]) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-piper-http.py b/examples/update-settings/tts/tts-piper-http.py index 685f81dd5..ed943b5b0 100644 --- a/examples/update-settings/tts/tts-piper-http.py +++ b/examples/update-settings/tts/tts-piper-http.py @@ -53,16 +53,16 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = PiperHttpTTSService( - base_url=os.getenv("PIPER_BASE_URL"), + base_url=os.environ["PIPER_BASE_URL"], aiohttp_session=session, settings=PiperHttpTTSService.Settings(voice="en_US-ryan-high"), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-piper.py b/examples/update-settings/tts/tts-piper.py index df10d999a..710a9ad7f 100644 --- a/examples/update-settings/tts/tts-piper.py +++ b/examples/update-settings/tts/tts-piper.py @@ -50,7 +50,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = PiperTTSService( settings=PiperTTSService.Settings( @@ -59,7 +59,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-resembleai.py b/examples/update-settings/tts/tts-resembleai.py index ae295cfa2..3f3a5e271 100644 --- a/examples/update-settings/tts/tts-resembleai.py +++ b/examples/update-settings/tts/tts-resembleai.py @@ -50,15 +50,15 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = ResembleAITTSService( - api_key=os.getenv("RESEMBLE_API_KEY"), - settings=ResembleAITTSService.Settings(voice=os.getenv("RESEMBLE_VOICE_UUID")), + api_key=os.environ["RESEMBLE_API_KEY"], + settings=ResembleAITTSService.Settings(voice=os.environ["RESEMBLE_VOICE_UUID"]), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), @@ -103,7 +103,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info("Updating ResembleAI TTS settings: voice (changed)") await task.queue_frame( TTSUpdateSettingsFrame( - delta=ResembleAITTSService.Settings(voice=os.getenv("RESEMBLE_VOICE_UUID_ALT")) + delta=ResembleAITTSService.Settings(voice=os.environ["RESEMBLE_VOICE_UUID_ALT"]) ) ) diff --git a/examples/update-settings/tts/tts-rime-http.py b/examples/update-settings/tts/tts-rime-http.py index af69dcda5..3e5b2d2e7 100644 --- a/examples/update-settings/tts/tts-rime-http.py +++ b/examples/update-settings/tts/tts-rime-http.py @@ -54,16 +54,16 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = RimeHttpTTSService( - api_key=os.getenv("RIME_API_KEY"), + api_key=os.environ["RIME_API_KEY"], settings=RimeHttpTTSService.Settings(voice="eva"), aiohttp_session=session, ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-rime.py b/examples/update-settings/tts/tts-rime.py index 44bc69e13..314eeeb05 100644 --- a/examples/update-settings/tts/tts-rime.py +++ b/examples/update-settings/tts/tts-rime.py @@ -50,15 +50,15 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = RimeTTSService( - api_key=os.getenv("RIME_API_KEY"), + api_key=os.environ["RIME_API_KEY"], settings=RimeTTSService.Settings(voice="luna"), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-sarvam-http.py b/examples/update-settings/tts/tts-sarvam-http.py index e9a35f8cd..ec6911f5f 100644 --- a/examples/update-settings/tts/tts-sarvam-http.py +++ b/examples/update-settings/tts/tts-sarvam-http.py @@ -54,12 +54,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) - tts = SarvamHttpTTSService(api_key=os.getenv("SARVAM_API_KEY"), aiohttp_session=session) + tts = SarvamHttpTTSService(api_key=os.environ["SARVAM_API_KEY"], aiohttp_session=session) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-sarvam.py b/examples/update-settings/tts/tts-sarvam.py index 77bb80d57..9cb24ad93 100644 --- a/examples/update-settings/tts/tts-sarvam.py +++ b/examples/update-settings/tts/tts-sarvam.py @@ -50,12 +50,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) - tts = SarvamTTSService(api_key=os.getenv("SARVAM_API_KEY")) + tts = SarvamTTSService(api_key=os.environ["SARVAM_API_KEY"]) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-speechmatics.py b/examples/update-settings/tts/tts-speechmatics.py index 4cefa1cc8..a4515739c 100644 --- a/examples/update-settings/tts/tts-speechmatics.py +++ b/examples/update-settings/tts/tts-speechmatics.py @@ -51,16 +51,16 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) async with aiohttp.ClientSession() as session: tts = SpeechmaticsTTSService( - api_key=os.getenv("SPEECHMATICS_API_KEY"), + api_key=os.environ["SPEECHMATICS_API_KEY"], aiohttp_session=session, ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-xtts.py b/examples/update-settings/tts/tts-xtts.py index b1fb4f309..0145e8c64 100644 --- a/examples/update-settings/tts/tts-xtts.py +++ b/examples/update-settings/tts/tts-xtts.py @@ -53,7 +53,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = XTTSService( aiohttp_session=session, @@ -64,7 +64,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/video-avatar/video-avatar-heygen-transport.py b/examples/video-avatar/video-avatar-heygen-transport.py index 47ec01abd..0ea504520 100644 --- a/examples/video-avatar/video-avatar-heygen-transport.py +++ b/examples/video-avatar/video-avatar-heygen-transport.py @@ -37,7 +37,7 @@ logger.add(sys.stderr, level="DEBUG") async def main(): async with aiohttp.ClientSession() as session: transport = HeyGenTransport( - api_key=os.getenv("HEYGEN_LIVE_AVATAR_API_KEY"), + api_key=os.environ["HEYGEN_LIVE_AVATAR_API_KEY"], service_type=ServiceType.LIVE_AVATAR, session=session, params=HeyGenParams( @@ -52,17 +52,17 @@ async def main(): ), ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="00967b2f-88a6-4a31-8153-110a92134b9f", ), ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( system_instruction="You are a helpful assistant. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Be succinct and respond to what the user said in a creative and helpful way.", ), diff --git a/examples/video-avatar/video-avatar-heygen-video-service.py b/examples/video-avatar/video-avatar-heygen-video-service.py index 35d2e3221..a48c42758 100644 --- a/examples/video-avatar/video-avatar-heygen-video-service.py +++ b/examples/video-avatar/video-avatar-heygen-video-service.py @@ -59,24 +59,24 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="00967b2f-88a6-4a31-8153-110a92134b9f", ), ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( system_instruction="You are a helpful assistant. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Be succinct and respond to what the user said in a creative and helpful way.", ), ) heyGen = HeyGenVideoService( - api_key=os.getenv("HEYGEN_LIVE_AVATAR_API_KEY"), + api_key=os.environ["HEYGEN_LIVE_AVATAR_API_KEY"], service_type=ServiceType.LIVE_AVATAR, session=session, session_request=LiveAvatarNewSessionRequest( diff --git a/examples/video-avatar/video-avatar-lemonslice-transport.py b/examples/video-avatar/video-avatar-lemonslice-transport.py index 52cd1ca25..08dd48aa3 100644 --- a/examples/video-avatar/video-avatar-lemonslice-transport.py +++ b/examples/video-avatar/video-avatar-lemonslice-transport.py @@ -41,7 +41,7 @@ async def main(): async with aiohttp.ClientSession() as session: transport = LemonSliceTransport( bot_name="Pipecat", - api_key=os.getenv("LEMONSLICE_API_KEY"), + api_key=os.environ["LEMONSLICE_API_KEY"], session=session, session_request=LemonSliceNewSessionRequest( agent_id=os.getenv("LEMONSLICE_AGENT_ID"), @@ -53,10 +53,10 @@ async def main(): ), ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) llm = GroqLLMService( - api_key=os.getenv("GROQ_API_KEY"), + api_key=os.environ["GROQ_API_KEY"], settings=GroqLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/video-avatar/video-avatar-simli-video-service.py b/examples/video-avatar/video-avatar-simli-video-service.py index 76c2dd99d..c91623253 100644 --- a/examples/video-avatar/video-avatar-simli-video-service.py +++ b/examples/video-avatar/video-avatar-simli-video-service.py @@ -56,22 +56,22 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", ), ) simli_ai = SimliVideoService( - api_key=os.getenv("SIMLI_API_KEY"), + api_key=os.environ["SIMLI_API_KEY"], face_id="cace3ef7-a4c4-425d-a8cf-a5358eb0c427", ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/video-avatar/video-avatar-tavus-transport.py b/examples/video-avatar/video-avatar-tavus-transport.py index db19e650e..566538dbb 100644 --- a/examples/video-avatar/video-avatar-tavus-transport.py +++ b/examples/video-avatar/video-avatar-tavus-transport.py @@ -37,8 +37,8 @@ async def main(): async with aiohttp.ClientSession() as session: transport = TavusTransport( bot_name="Pipecat bot", - api_key=os.getenv("TAVUS_API_KEY"), - replica_id=os.getenv("TAVUS_REPLICA_ID"), + api_key=os.environ["TAVUS_API_KEY"], + replica_id=os.environ["TAVUS_REPLICA_ID"], session=session, params=TavusParams( audio_in_enabled=True, @@ -47,17 +47,17 @@ async def main(): ), ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="a167e0f3-df7e-4d52-a9c3-f949145efdab", ), ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/video-avatar/video-avatar-tavus-video-service.py b/examples/video-avatar/video-avatar-tavus-video-service.py index 5658951b5..ce3075074 100644 --- a/examples/video-avatar/video-avatar-tavus-video-service.py +++ b/examples/video-avatar/video-avatar-tavus-video-service.py @@ -59,25 +59,25 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="a167e0f3-df7e-4d52-a9c3-f949145efdab", ), ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), ) tavus = TavusVideoService( - api_key=os.getenv("TAVUS_API_KEY"), - replica_id=os.getenv("TAVUS_REPLICA_ID"), + api_key=os.environ["TAVUS_API_KEY"], + replica_id=os.environ["TAVUS_REPLICA_ID"], session=session, ) diff --git a/examples/video-processing/video-processing.py b/examples/video-processing/video-processing.py index 531c7c940..812de5b03 100644 --- a/examples/video-processing/video-processing.py +++ b/examples/video-processing/video-processing.py @@ -96,20 +96,13 @@ Respond to what the user said in a creative and helpful way. Keep your responses async def run_bot(pipecat_transport): llm = GeminiLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck transcribe_user_audio=True, system_instruction=SYSTEM_INSTRUCTION, ) - messages = [ - { - "role": "developer", - "content": "Start by greeting the user warmly and introducing yourself.", - } - ] - - context = LLMContext(messages) + context = LLMContext() user_aggregator, assistant_aggregator = LLMContextAggregatorPair( context, user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), @@ -141,6 +134,12 @@ async def run_bot(pipecat_transport): async def on_client_ready(rtvi): logger.info("Pipecat client ready.") # Kick off the conversation. + context.add_message( + { + "role": "developer", + "content": "Start by greeting the user warmly and introducing yourself.", + } + ) await task.queue_frames([LLMRunFrame()]) @pipecat_transport.event_handler("on_client_connected") diff --git a/examples/vision/vision-anthropic.py b/examples/vision/vision-anthropic.py index 642885dcf..d02cf8b95 100644 --- a/examples/vision/vision-anthropic.py +++ b/examples/vision/vision-anthropic.py @@ -49,17 +49,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), + api_key=os.environ["ANTHROPIC_API_KEY"], settings=AnthropicLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You are also able to describe images.", ), diff --git a/examples/vision/vision-aws.py b/examples/vision/vision-aws.py index 47d2b7970..8905f1e9f 100644 --- a/examples/vision/vision-aws.py +++ b/examples/vision/vision-aws.py @@ -49,10 +49,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), diff --git a/examples/vision/vision-gemini-flash.py b/examples/vision/vision-gemini-flash.py index 248bc08a7..b94942766 100644 --- a/examples/vision/vision-gemini-flash.py +++ b/examples/vision/vision-gemini-flash.py @@ -49,17 +49,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You are also able to describe images.", ), diff --git a/examples/vision/vision-moondream.py b/examples/vision/vision-moondream.py index 6a73de7b3..5b2a3f2e7 100644 --- a/examples/vision/vision-moondream.py +++ b/examples/vision/vision-moondream.py @@ -41,7 +41,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), diff --git a/examples/vision/vision-openai-responses-http.py b/examples/vision/vision-openai-responses-http.py index 866a86d2c..86c63a266 100644 --- a/examples/vision/vision-openai-responses-http.py +++ b/examples/vision/vision-openai-responses-http.py @@ -49,17 +49,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesHttpLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIResponsesHttpLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You are also able to describe images.", ), diff --git a/examples/vision/vision-openai-responses.py b/examples/vision/vision-openai-responses.py index a3c113cb2..d66c2b2ba 100644 --- a/examples/vision/vision-openai-responses.py +++ b/examples/vision/vision-openai-responses.py @@ -49,17 +49,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIResponsesLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You are also able to describe images.", ), diff --git a/examples/vision/vision-openai.py b/examples/vision/vision-openai.py index 8c8af6352..2dfede380 100644 --- a/examples/vision/vision-openai.py +++ b/examples/vision/vision-openai.py @@ -49,17 +49,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You are also able to describe images.", ), diff --git a/examples/voice/voice-aicoustics.py b/examples/voice/voice-aicoustics.py index 16b850052..39110de88 100644 --- a/examples/voice/voice-aicoustics.py +++ b/examples/voice/voice-aicoustics.py @@ -73,17 +73,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-assemblyai-turn-detection.py b/examples/voice/voice-assemblyai-turn-detection.py index 64d257675..26662872e 100644 --- a/examples/voice/voice-assemblyai-turn-detection.py +++ b/examples/voice/voice-assemblyai-turn-detection.py @@ -91,7 +91,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = AssemblyAISTTService( - api_key=os.getenv("ASSEMBLYAI_API_KEY"), + api_key=os.environ["ASSEMBLYAI_API_KEY"], vad_force_turn_endpoint=False, # Use AssemblyAI's built-in turn detection settings=AssemblyAISTTService.Settings( model="u3-rt-pro", @@ -106,14 +106,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-assemblyai.py b/examples/voice/voice-assemblyai.py index 6bcae005a..826b4b020 100644 --- a/examples/voice/voice-assemblyai.py +++ b/examples/voice/voice-assemblyai.py @@ -54,18 +54,18 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = AssemblyAISTTService( - api_key=os.getenv("ASSEMBLYAI_API_KEY"), + api_key=os.environ["ASSEMBLYAI_API_KEY"], ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-asyncai-http.py b/examples/voice/voice-asyncai-http.py index b861b32ad..480ff9bfd 100644 --- a/examples/voice/voice-asyncai-http.py +++ b/examples/voice/voice-asyncai-http.py @@ -56,7 +56,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Create an HTTP session async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = AsyncAIHttpTTSService( api_key=os.getenv("ASYNCAI_API_KEY", ""), @@ -67,7 +67,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-asyncai.py b/examples/voice/voice-asyncai.py index 55daf5e93..46240542c 100644 --- a/examples/voice/voice-asyncai.py +++ b/examples/voice/voice-asyncai.py @@ -53,7 +53,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = AsyncAITTSService( api_key=os.getenv("ASYNCAI_API_KEY", ""), @@ -63,7 +63,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-azure-http.py b/examples/voice/voice-azure-http.py index 6d8206910..c3872cbfb 100644 --- a/examples/voice/voice-azure-http.py +++ b/examples/voice/voice-azure-http.py @@ -53,18 +53,18 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = AzureSTTService( - api_key=os.getenv("AZURE_SPEECH_API_KEY"), + api_key=os.environ["AZURE_SPEECH_API_KEY"], region=os.getenv("AZURE_SPEECH_REGION"), ) tts = AzureHttpTTSService( - api_key=os.getenv("AZURE_SPEECH_API_KEY"), - region=os.getenv("AZURE_SPEECH_REGION"), + api_key=os.environ["AZURE_SPEECH_API_KEY"], + region=os.environ["AZURE_SPEECH_REGION"], ) llm = AzureLLMService( - api_key=os.getenv("AZURE_CHATGPT_API_KEY"), - endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + api_key=os.environ["AZURE_CHATGPT_API_KEY"], + endpoint=os.environ["AZURE_CHATGPT_ENDPOINT"], settings=AzureLLMService.Settings( model=os.getenv("AZURE_CHATGPT_MODEL"), system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/voice/voice-azure.py b/examples/voice/voice-azure.py index a7977b04b..708c96e18 100644 --- a/examples/voice/voice-azure.py +++ b/examples/voice/voice-azure.py @@ -53,18 +53,18 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = AzureSTTService( - api_key=os.getenv("AZURE_SPEECH_API_KEY"), + api_key=os.environ["AZURE_SPEECH_API_KEY"], region=os.getenv("AZURE_SPEECH_REGION"), ) tts = AzureTTSService( - api_key=os.getenv("AZURE_SPEECH_API_KEY"), - region=os.getenv("AZURE_SPEECH_REGION"), + api_key=os.environ["AZURE_SPEECH_API_KEY"], + region=os.environ["AZURE_SPEECH_REGION"], ) llm = AzureLLMService( - api_key=os.getenv("AZURE_CHATGPT_API_KEY"), - endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + api_key=os.environ["AZURE_CHATGPT_API_KEY"], + endpoint=os.environ["AZURE_CHATGPT_ENDPOINT"], settings=AzureLLMService.Settings( model=os.getenv("AZURE_CHATGPT_MODEL"), system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/voice/voice-camb.py b/examples/voice/voice-camb.py index 90b6cabd4..a84b3c3ec 100644 --- a/examples/voice/voice-camb.py +++ b/examples/voice/voice-camb.py @@ -52,17 +52,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info("Starting Camb AI TTS bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CambTTSService( - api_key=os.getenv("CAMB_API_KEY"), + api_key=os.environ["CAMB_API_KEY"], settings=CambTTSService.Settings( model="mars-flash", ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful voice assistant powered by Camb AI text-to-speech. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Keep responses concise.", ), diff --git a/examples/voice/voice-cartesia-http.py b/examples/voice/voice-cartesia-http.py index 15fe51142..f2f02ec7d 100644 --- a/examples/voice/voice-cartesia-http.py +++ b/examples/voice/voice-cartesia-http.py @@ -54,10 +54,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = CartesiaSTTService(api_key=os.getenv("CARTESIA_API_KEY")) + stt = CartesiaSTTService(api_key=os.environ["CARTESIA_API_KEY"]) tts = CartesiaHttpTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], aiohttp_session=session, settings=CartesiaHttpTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady @@ -65,7 +65,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-cartesia.py b/examples/voice/voice-cartesia.py index 0489663e7..2a399b954 100644 --- a/examples/voice/voice-cartesia.py +++ b/examples/voice/voice-cartesia.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = CartesiaSTTService(api_key=os.getenv("CARTESIA_API_KEY")) + stt = CartesiaSTTService(api_key=os.environ["CARTESIA_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-deepgram-flux-sagemaker.py b/examples/voice/voice-deepgram-flux-sagemaker.py index c203ed43a..c0426ab15 100644 --- a/examples/voice/voice-deepgram-flux-sagemaker.py +++ b/examples/voice/voice-deepgram-flux-sagemaker.py @@ -59,8 +59,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # - AWS credentials configured (via environment variables or AWS CLI) # - A deployed SageMaker endpoint with Deepgram Flux model stt = DeepgramFluxSageMakerSTTService( - endpoint_name=os.getenv("SAGEMAKER_STT_ENDPOINT_NAME"), - region=os.getenv("AWS_REGION"), + endpoint_name=os.environ["SAGEMAKER_STT_ENDPOINT_NAME"], + region=os.environ["AWS_REGION"], settings=DeepgramFluxSageMakerSTTService.Settings( min_confidence=0.3, ), @@ -71,8 +71,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # - AWS credentials configured (via environment variables or AWS CLI) # - A deployed SageMaker endpoint with Deepgram TTS model tts = DeepgramSageMakerTTSService( - endpoint_name=os.getenv("SAGEMAKER_TTS_ENDPOINT_NAME"), - region=os.getenv("AWS_REGION"), + endpoint_name=os.environ["SAGEMAKER_TTS_ENDPOINT_NAME"], + region=os.environ["AWS_REGION"], settings=DeepgramSageMakerTTSService.Settings( voice="aura-2-andromeda-en", ), diff --git a/examples/voice/voice-deepgram-flux.py b/examples/voice/voice-deepgram-flux.py index 76a5bb783..7e9ec3938 100644 --- a/examples/voice/voice-deepgram-flux.py +++ b/examples/voice/voice-deepgram-flux.py @@ -55,21 +55,21 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = DeepgramFluxSTTService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], settings=DeepgramFluxSTTService.Settings( min_confidence=0.3, ), ) tts = DeepgramTTSService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], settings=DeepgramTTSService.Settings( voice="aura-2-andromeda-en", ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-deepgram-http.py b/examples/voice/voice-deepgram-http.py index c81b2cd2f..d53255ca0 100644 --- a/examples/voice/voice-deepgram-http.py +++ b/examples/voice/voice-deepgram-http.py @@ -55,10 +55,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = DeepgramHttpTTSService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], settings=DeepgramHttpTTSService.Settings( voice="aura-2-andromeda-en", ), @@ -66,7 +66,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-deepgram-sagemaker.py b/examples/voice/voice-deepgram-sagemaker.py index e823f18f9..30e861518 100644 --- a/examples/voice/voice-deepgram-sagemaker.py +++ b/examples/voice/voice-deepgram-sagemaker.py @@ -58,8 +58,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # - AWS credentials configured (via environment variables or AWS CLI) # - A deployed SageMaker endpoint with Deepgram model stt = DeepgramSageMakerSTTService( - endpoint_name=os.getenv("SAGEMAKER_STT_ENDPOINT_NAME"), - region=os.getenv("AWS_REGION"), + endpoint_name=os.environ["SAGEMAKER_STT_ENDPOINT_NAME"], + region=os.environ["AWS_REGION"], ) # Initialize Deepgram SageMaker TTS Service @@ -67,8 +67,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # - AWS credentials configured (via environment variables or AWS CLI) # - A deployed SageMaker endpoint with Deepgram TTS model tts = DeepgramSageMakerTTSService( - endpoint_name=os.getenv("SAGEMAKER_TTS_ENDPOINT_NAME"), - region=os.getenv("AWS_REGION"), + endpoint_name=os.environ["SAGEMAKER_TTS_ENDPOINT_NAME"], + region=os.environ["AWS_REGION"], settings=DeepgramSageMakerTTSService.Settings( voice="aura-2-andromeda-en", ), diff --git a/examples/voice/voice-deepgram-vad.py b/examples/voice/voice-deepgram-vad.py deleted file mode 100644 index bbdb01efe..000000000 --- a/examples/voice/voice-deepgram-vad.py +++ /dev/null @@ -1,133 +0,0 @@ -# -# Copyright (c) 2024-2026, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - - -import os - -from dotenv import load_dotenv -from loguru import logger - -from pipecat.frames.frames import LLMRunFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import ( - LLMContextAggregatorPair, - LLMUserAggregatorParams, -) -from pipecat.runner.types import RunnerArguments -from pipecat.runner.utils import create_transport -from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.services.deepgram.tts import DeepgramTTSService -from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.transports.daily.transport import DailyParams -from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams -from pipecat.turns.user_turn_strategies import ExternalUserTurnStrategies - -load_dotenv(override=True) - - -# We use lambdas to defer transport parameter creation until the transport -# type is selected at runtime. -transport_params = { - "daily": lambda: DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - ), - "twilio": lambda: FastAPIWebsocketParams( - audio_in_enabled=True, - audio_out_enabled=True, - ), - "webrtc": lambda: TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - ), -} - - -async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - logger.info(f"Starting bot") - - stt = DeepgramSTTService( - api_key=os.getenv("DEEPGRAM_API_KEY"), - settings=DeepgramSTTService.Settings( - vad_events=True, - utterance_end_ms="1000", - ), - ) - - tts = DeepgramTTSService( - api_key=os.getenv("DEEPGRAM_API_KEY"), - settings=DeepgramTTSService.Settings( - voice="aura-2-andromeda-en", - ), - ) - - llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), - settings=OpenAILLMService.Settings( - system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", - ), - ) - - context = LLMContext() - user_aggregator, assistant_aggregator = LLMContextAggregatorPair( - context, - user_params=LLMUserAggregatorParams(user_turn_strategies=ExternalUserTurnStrategies()), - ) - - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, # STT - user_aggregator, # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - assistant_aggregator, # Assistant spoken responses - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - enable_usage_metrics=True, - ), - idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - ) - - @transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - logger.info(f"Client connected") - # Kick off the conversation. - context.add_message( - {"role": "developer", "content": "Please introduce yourself to the user."} - ) - await task.queue_frames([LLMRunFrame()]) - - @transport.event_handler("on_client_disconnected") - async def on_client_disconnected(transport, client): - logger.info(f"Client disconnected") - await task.cancel() - - runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) - - await runner.run(task) - - -async def bot(runner_args: RunnerArguments): - """Main bot entry point compatible with Pipecat Cloud.""" - transport = await create_transport(runner_args, transport_params) - await run_bot(transport, runner_args) - - -if __name__ == "__main__": - from pipecat.runner.run import main - - main() diff --git a/examples/voice/voice-deepgram.py b/examples/voice/voice-deepgram.py index bebde3fe3..323fd34a3 100644 --- a/examples/voice/voice-deepgram.py +++ b/examples/voice/voice-deepgram.py @@ -53,17 +53,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = DeepgramTTSService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], settings=DeepgramTTSService.Settings( voice="aura-2-andromeda-en", ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-elevenlabs-http.py b/examples/voice/voice-elevenlabs-http.py index b9e35ec4f..87e8dc2ca 100644 --- a/examples/voice/voice-elevenlabs-http.py +++ b/examples/voice/voice-elevenlabs-http.py @@ -57,7 +57,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Create an HTTP session async with aiohttp.ClientSession() as session: stt = ElevenLabsSTTService( - api_key=os.getenv("ELEVENLABS_API_KEY"), + api_key=os.environ["ELEVENLABS_API_KEY"], aiohttp_session=session, ) @@ -70,7 +70,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-elevenlabs.py b/examples/voice/voice-elevenlabs.py index ebdfdde72..8f61bb665 100644 --- a/examples/voice/voice-elevenlabs.py +++ b/examples/voice/voice-elevenlabs.py @@ -53,7 +53,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = ElevenLabsRealtimeSTTService(api_key=os.getenv("ELEVENLABS_API_KEY")) + stt = ElevenLabsRealtimeSTTService(api_key=os.environ["ELEVENLABS_API_KEY"]) tts = ElevenLabsTTSService( api_key=os.getenv("ELEVENLABS_API_KEY", ""), @@ -63,7 +63,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-fal.py b/examples/voice/voice-fal.py index 5ce43bbe6..d4b1e4a52 100644 --- a/examples/voice/voice-fal.py +++ b/examples/voice/voice-fal.py @@ -61,14 +61,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-fish.py b/examples/voice/voice-fish.py index 5519b6c4f..0ff8503c8 100644 --- a/examples/voice/voice-fish.py +++ b/examples/voice/voice-fish.py @@ -53,17 +53,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = FishAudioTTSService( - api_key=os.getenv("FISH_API_KEY"), + api_key=os.environ["FISH_API_KEY"], settings=FishAudioTTSService.Settings( voice="4ce7e917cedd4bc2bb2e6ff3a46acaa1", # Barack Obama ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-gladia-vad.py b/examples/voice/voice-gladia-vad.py index f29930c3e..c0eea8841 100644 --- a/examples/voice/voice-gladia-vad.py +++ b/examples/voice/voice-gladia-vad.py @@ -55,9 +55,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") + region = os.getenv("GLADIA_REGION", "us-west") + assert region in ("us-west", "eu-west"), f"Invalid GLADIA_REGION: {region}" + stt = GladiaSTTService( - api_key=os.getenv("GLADIA_API_KEY", ""), - region=os.getenv("GLADIA_REGION"), + api_key=os.environ["GLADIA_API_KEY"], + region=region, settings=GladiaSTTService.Settings( language_config=LanguageConfig( languages=[Language.EN], diff --git a/examples/voice/voice-gladia.py b/examples/voice/voice-gladia.py index 3f417bf5a..a6a909a9a 100644 --- a/examples/voice/voice-gladia.py +++ b/examples/voice/voice-gladia.py @@ -54,9 +54,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") + region = os.getenv("GLADIA_REGION", "us-west") + assert region in ("us-west", "eu-west"), f"Invalid GLADIA_REGION: {region}" + stt = GladiaSTTService( - api_key=os.getenv("GLADIA_API_KEY", ""), - region=os.getenv("GLADIA_REGION"), + api_key=os.environ["GLADIA_API_KEY"], + region=region, settings=GladiaSTTService.Settings( language_config=LanguageConfig( languages=[Language.EN], diff --git a/examples/voice/voice-google-audio-in.py b/examples/voice/voice-google-audio-in.py index fa314afed..f6a350411 100644 --- a/examples/voice/voice-google-audio-in.py +++ b/examples/voice/voice-google-audio-in.py @@ -215,7 +215,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( model="gemini-2.5-flash", system_instruction=system_message, @@ -230,7 +230,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): language=Language.EN_US, ), params=GoogleTTSService.InputParams(language=Language.EN_US), - credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + credentials=os.environ["GOOGLE_TEST_CREDENTIALS"], ) context = LLMContext() diff --git a/examples/voice/voice-google-gemini-tts.py b/examples/voice/voice-google-gemini-tts.py index 7b995a564..8aa7e3c39 100644 --- a/examples/voice/voice-google-gemini-tts.py +++ b/examples/voice/voice-google-gemini-tts.py @@ -57,11 +57,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): settings=GoogleSTTService.Settings( languages=[Language.EN_US], ), - credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + credentials=os.environ["GOOGLE_TEST_CREDENTIALS"], ) tts = GeminiTTSService( - credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + credentials=os.environ["GOOGLE_TEST_CREDENTIALS"], settings=GeminiTTSService.Settings( model="gemini-2.5-flash-tts", voice="Charon", @@ -71,7 +71,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], model="gemini-2.5-flash", settings=GoogleLLMService.Settings( system_instruction="""You are a helpful assistant in a voice conversation. diff --git a/examples/voice/voice-google-http.py b/examples/voice/voice-google-http.py index d961f2d8d..0e642b8ca 100644 --- a/examples/voice/voice-google-http.py +++ b/examples/voice/voice-google-http.py @@ -59,7 +59,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Add model to use a specific model # model="chirp_3", ), - credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + credentials=os.environ["GOOGLE_TEST_CREDENTIALS"], location="us", ) @@ -68,11 +68,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): voice="en-US-Chirp3-HD-Charon", language=Language.EN_US, ), - credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + credentials=os.environ["GOOGLE_TEST_CREDENTIALS"], ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( model="gemini-2.5-flash", # force a certain amount of thinking if you want it diff --git a/examples/voice/voice-google-image.py b/examples/voice/voice-google-image.py index 80f115390..52949b357 100644 --- a/examples/voice/voice-google-image.py +++ b/examples/voice/voice-google-image.py @@ -67,14 +67,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = GoogleSTTService( - credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + credentials=os.environ["GOOGLE_TEST_CREDENTIALS"], settings=GoogleSTTService.Settings( languages=[Language.EN_US], ), ) tts = GoogleTTSService( - credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + credentials=os.environ["GOOGLE_TEST_CREDENTIALS"], settings=GoogleTTSService.Settings( voice="en-US-Chirp3-HD-Charon", language=Language.EN_US, @@ -82,7 +82,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( model="gemini-2.5-flash-image", # model="gemini-3-pro-image-preview", # A more powerful model, but slower, diff --git a/examples/voice/voice-google.py b/examples/voice/voice-google.py index 10684b8cc..9f6a43874 100644 --- a/examples/voice/voice-google.py +++ b/examples/voice/voice-google.py @@ -59,7 +59,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Add model to use a specific model # model="chirp_3", ), - credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + credentials=os.environ["GOOGLE_TEST_CREDENTIALS"], location="us", ) @@ -68,11 +68,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): voice="en-US-Chirp3-HD-Charon", language=Language.EN_US, ), - credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + credentials=os.environ["GOOGLE_TEST_CREDENTIALS"], ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( model="gemini-2.5-flash", # force a certain amount of thinking if you want it diff --git a/examples/voice/voice-gradium.py b/examples/voice/voice-gradium.py index 3573d0b39..89573f48a 100644 --- a/examples/voice/voice-gradium.py +++ b/examples/voice/voice-gradium.py @@ -53,7 +53,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = GradiumSTTService( - api_key=os.getenv("GRADIUM_API_KEY"), + api_key=os.environ["GRADIUM_API_KEY"], api_endpoint_base_url="wss://us.api.gradium.ai/api/speech/asr", settings=GradiumSTTService.Settings( language=Language.EN, @@ -61,7 +61,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = GradiumTTSService( - api_key=os.getenv("GRADIUM_API_KEY"), + api_key=os.environ["GRADIUM_API_KEY"], url="wss://us.api.gradium.ai/api/speech/tts", settings=GradiumTTSService.Settings( voice="YTpq7expH9539ERJ", @@ -69,7 +69,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-groq.py b/examples/voice/voice-groq.py index 7ffa8b06b..9b1b34da4 100644 --- a/examples/voice/voice-groq.py +++ b/examples/voice/voice-groq.py @@ -52,17 +52,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = GroqSTTService(api_key=os.getenv("GROQ_API_KEY")) + stt = GroqSTTService(api_key=os.environ["GROQ_API_KEY"]) llm = GroqLLMService( - api_key=os.getenv("GROQ_API_KEY"), + api_key=os.environ["GROQ_API_KEY"], settings=GroqLLMService.Settings( model="llama-3.1-8b-instant", system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), ) - tts = GroqTTSService(api_key=os.getenv("GROQ_API_KEY")) + tts = GroqTTSService(api_key=os.environ["GROQ_API_KEY"]) context = LLMContext() user_aggregator, assistant_aggregator = LLMContextAggregatorPair( diff --git a/examples/voice/voice-hume.py b/examples/voice/voice-hume.py index 2dbeeb924..bda68d6f9 100644 --- a/examples/voice/voice-hume.py +++ b/examples/voice/voice-hume.py @@ -54,7 +54,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = HumeTTSService( api_key=os.getenv("HUME_API_KEY"), @@ -65,7 +65,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-inworld-http.py b/examples/voice/voice-inworld-http.py index 3d06a055a..2a388f62b 100644 --- a/examples/voice/voice-inworld-http.py +++ b/examples/voice/voice-inworld-http.py @@ -53,7 +53,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info("Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = InworldHttpTTSService( api_key=os.getenv("INWORLD_API_KEY", ""), @@ -66,7 +66,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful AI demonstrating Inworld AI's TTS. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a friendly and helpful way.", ), diff --git a/examples/voice/voice-inworld.py b/examples/voice/voice-inworld.py index 7db6edbee..b8603efc9 100644 --- a/examples/voice/voice-inworld.py +++ b/examples/voice/voice-inworld.py @@ -50,7 +50,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info("Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = InworldTTSService( api_key=os.getenv("INWORLD_API_KEY", ""), @@ -61,7 +61,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful AI demonstrating Inworld AI's TTS. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a friendly and helpful way.", ), diff --git a/examples/voice/voice-kokoro.py b/examples/voice/voice-kokoro.py index c55c2ba08..fe74589dc 100644 --- a/examples/voice/voice-kokoro.py +++ b/examples/voice/voice-kokoro.py @@ -52,7 +52,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = KokoroTTSService( settings=KokoroTTSService.Settings( @@ -61,7 +61,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-krisp-viva.py b/examples/voice/voice-krisp-viva.py index 06a236e0d..fa6498616 100644 --- a/examples/voice/voice-krisp-viva.py +++ b/examples/voice/voice-krisp-viva.py @@ -84,17 +84,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-langchain.py b/examples/voice/voice-langchain.py index 8ab17aa79..19fda0c91 100644 --- a/examples/voice/voice-langchain.py +++ b/examples/voice/voice-langchain.py @@ -16,7 +16,7 @@ from langchain_openai import ChatOpenAI from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import LLMMessagesUpdateFrame +from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -67,10 +67,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -129,8 +129,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # An `LLMContextFrame` will be picked up by the LangchainProcessor using # only the content of the last message to inject it in the prompt defined # above. So no role is required here. - messages = [({"content": "Please briefly introduce yourself to the user."})] - await task.queue_frames([LLMMessagesUpdateFrame(messages, run_llm=True)]) + context.add_message( + {"role": "developer", "content": "Please briefly introduce yourself to the user."} + ) + await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): diff --git a/examples/voice/voice-lmnt.py b/examples/voice/voice-lmnt.py index 41e42a961..6faa4c31c 100644 --- a/examples/voice/voice-lmnt.py +++ b/examples/voice/voice-lmnt.py @@ -52,17 +52,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = LmntTTSService( - api_key=os.getenv("LMNT_API_KEY"), + api_key=os.environ["LMNT_API_KEY"], settings=LmntTTSService.Settings( voice="morgan", ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-minimax.py b/examples/voice/voice-minimax.py index 92fda761d..9e4291af7 100644 --- a/examples/voice/voice-minimax.py +++ b/examples/voice/voice-minimax.py @@ -57,7 +57,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Create an HTTP session async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = MiniMaxHttpTTSService( api_key=os.getenv("MINIMAX_API_KEY", ""), @@ -69,7 +69,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-mistral.py b/examples/voice/voice-mistral.py index 440039b65..4664caf22 100644 --- a/examples/voice/voice-mistral.py +++ b/examples/voice/voice-mistral.py @@ -53,17 +53,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = MistralSTTService(api_key=os.getenv("MISTRAL_API_KEY")) + stt = MistralSTTService(api_key=os.environ["MISTRAL_API_KEY"]) tts = MistralTTSService( - api_key=os.getenv("MISTRAL_API_KEY"), + api_key=os.environ["MISTRAL_API_KEY"], settings=MistralTTSService.Settings( voice="c69964a6-ab8b-4f8a-9465-ec0925096ec8", ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-neuphonic-http.py b/examples/voice/voice-neuphonic-http.py index 6392d4606..9cbcc8ede 100644 --- a/examples/voice/voice-neuphonic-http.py +++ b/examples/voice/voice-neuphonic-http.py @@ -56,10 +56,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Create an HTTP session async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = NeuphonicHttpTTSService( - api_key=os.getenv("NEUPHONIC_API_KEY"), + api_key=os.environ["NEUPHONIC_API_KEY"], settings=NeuphonicHttpTTSService.Settings( voice="fc854436-2dac-4d21-aa69-ae17b54e98eb", # Emily ), @@ -67,7 +67,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-neuphonic.py b/examples/voice/voice-neuphonic.py index a0aaf82fa..1ea90fd51 100644 --- a/examples/voice/voice-neuphonic.py +++ b/examples/voice/voice-neuphonic.py @@ -52,17 +52,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = NeuphonicTTSService( - api_key=os.getenv("NEUPHONIC_API_KEY"), + api_key=os.environ["NEUPHONIC_API_KEY"], settings=NeuphonicTTSService.Settings( voice="fc854436-2dac-4d21-aa69-ae17b54e98eb", # Emily ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-nvidia.py b/examples/voice/voice-nvidia.py index b308f71a2..7f84134d3 100644 --- a/examples/voice/voice-nvidia.py +++ b/examples/voice/voice-nvidia.py @@ -52,17 +52,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = NvidiaSTTService(api_key=os.getenv("NVIDIA_API_KEY")) + stt = NvidiaSTTService(api_key=os.environ["NVIDIA_API_KEY"]) llm = NvidiaLLMService( - api_key=os.getenv("NVIDIA_API_KEY"), + api_key=os.environ["NVIDIA_API_KEY"], settings=NvidiaLLMService.Settings( model="meta/llama-3.3-70b-instruct", system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), ) - tts = NvidiaTTSService(api_key=os.getenv("NVIDIA_API_KEY")) + tts = NvidiaTTSService(api_key=os.environ["NVIDIA_API_KEY"]) context = LLMContext() user_aggregator, assistant_aggregator = LLMContextAggregatorPair( diff --git a/examples/voice/voice-openai-http.py b/examples/voice/voice-openai-http.py index 19f1a3e8a..1483acb69 100644 --- a/examples/voice/voice-openai-http.py +++ b/examples/voice/voice-openai-http.py @@ -53,7 +53,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = OpenAISTTService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAISTTService.Settings( model="gpt-4o-transcribe", prompt="Expect words related to dogs, such as breed names.", @@ -61,14 +61,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = OpenAITTSService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAITTSService.Settings( voice="ballad", ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are very knowledgable about dogs. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", ), diff --git a/examples/voice/voice-openai-responses-http.py b/examples/voice/voice-openai-responses-http.py index 6960bf7b2..b3f49cbe1 100644 --- a/examples/voice/voice-openai-responses-http.py +++ b/examples/voice/voice-openai-responses-http.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesHttpLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIResponsesHttpLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-openai-responses.py b/examples/voice/voice-openai-responses.py index baae3754a..d1ecc0946 100644 --- a/examples/voice/voice-openai-responses.py +++ b/examples/voice/voice-openai-responses.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIResponsesLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-openai.py b/examples/voice/voice-openai.py index b749ce27a..8f162671f 100644 --- a/examples/voice/voice-openai.py +++ b/examples/voice/voice-openai.py @@ -54,7 +54,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = OpenAIRealtimeSTTService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIRealtimeSTTService.Settings( model="gpt-4o-transcribe", prompt="Expect words related to dogs, such as breed names.", @@ -63,14 +63,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = OpenAITTSService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAITTSService.Settings( voice="ballad", ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are very knowledgable about dogs. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", ), diff --git a/examples/voice/voice-piper.py b/examples/voice/voice-piper.py index 617223ceb..dffdd4d34 100644 --- a/examples/voice/voice-piper.py +++ b/examples/voice/voice-piper.py @@ -52,7 +52,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = PiperTTSService( settings=PiperTTSService.Settings( @@ -61,7 +61,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-resemble.py b/examples/voice/voice-resemble.py index c27ca9255..f3c2a1336 100644 --- a/examples/voice/voice-resemble.py +++ b/examples/voice/voice-resemble.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = ResembleAITTSService( - api_key=os.getenv("RESEMBLE_API_KEY"), + api_key=os.environ["RESEMBLE_API_KEY"], settings=ResembleAITTSService.Settings( - voice=os.getenv("RESEMBLE_VOICE_UUID"), + voice=os.environ["RESEMBLE_VOICE_UUID"], ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-rime-http.py b/examples/voice/voice-rime-http.py index 165735d6e..72bc63ca7 100644 --- a/examples/voice/voice-rime-http.py +++ b/examples/voice/voice-rime-http.py @@ -56,7 +56,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Create an HTTP session async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = RimeHttpTTSService( api_key=os.getenv("RIME_API_KEY", ""), @@ -69,7 +69,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-rime.py b/examples/voice/voice-rime.py index 07ea429c7..bd1f50dbc 100644 --- a/examples/voice/voice-rime.py +++ b/examples/voice/voice-rime.py @@ -52,7 +52,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = RimeTTSService( api_key=os.getenv("RIME_API_KEY", ""), @@ -62,7 +62,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-sarvam-http.py b/examples/voice/voice-sarvam-http.py index bca27e051..b70f44e2a 100644 --- a/examples/voice/voice-sarvam-http.py +++ b/examples/voice/voice-sarvam-http.py @@ -58,14 +58,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Create an HTTP session async with aiohttp.ClientSession() as session: stt = SarvamSTTService( - api_key=os.getenv("SARVAM_API_KEY"), + api_key=os.environ["SARVAM_API_KEY"], settings=SarvamSTTService.Settings( model="saarika:v2.5", ), ) tts = SarvamHttpTTSService( - api_key=os.getenv("SARVAM_API_KEY"), + api_key=os.environ["SARVAM_API_KEY"], aiohttp_session=session, settings=SarvamHttpTTSService.Settings( language=Language.EN_IN, @@ -73,7 +73,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = SarvamLLMService( - api_key=os.getenv("SARVAM_API_KEY"), + api_key=os.environ["SARVAM_API_KEY"], settings=SarvamLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-sarvam.py b/examples/voice/voice-sarvam.py index feda4fe38..5c646d505 100644 --- a/examples/voice/voice-sarvam.py +++ b/examples/voice/voice-sarvam.py @@ -53,21 +53,21 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = SarvamSTTService( - api_key=os.getenv("SARVAM_API_KEY"), + api_key=os.environ["SARVAM_API_KEY"], settings=SarvamSTTService.Settings( model="saaras:v3", ), ) tts = SarvamTTSService( - api_key=os.getenv("SARVAM_API_KEY"), + api_key=os.environ["SARVAM_API_KEY"], settings=SarvamTTSService.Settings( model="bulbul:v3", voice="shubh", ), ) llm = SarvamLLMService( - api_key=os.getenv("SARVAM_API_KEY"), + api_key=os.environ["SARVAM_API_KEY"], settings=SarvamLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-smallest.py b/examples/voice/voice-smallest.py index f1b3dcf43..b5b82802a 100644 --- a/examples/voice/voice-smallest.py +++ b/examples/voice/voice-smallest.py @@ -52,21 +52,21 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = SmallestSTTService( - api_key=os.getenv("SMALLEST_API_KEY"), + api_key=os.environ["SMALLEST_API_KEY"], settings=SmallestSTTService.Settings( language=Language.EN, ), ) tts = SmallestTTSService( - api_key=os.getenv("SMALLEST_API_KEY"), + api_key=os.environ["SMALLEST_API_KEY"], settings=SmallestTTSService.Settings( voice="sophia", ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", ), diff --git a/examples/voice/voice-soniox.py b/examples/voice/voice-soniox.py index 857fd19e8..b89106d14 100644 --- a/examples/voice/voice-soniox.py +++ b/examples/voice/voice-soniox.py @@ -52,7 +52,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = SonioxSTTService( - api_key=os.getenv("SONIOX_API_KEY"), + api_key=os.environ["SONIOX_API_KEY"], settings=SonioxSTTService.Settings( # Add language hints to use a specific language # Add strict mode to enforce the language hints @@ -62,14 +62,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-speechmatics-vad.py b/examples/voice/voice-speechmatics-vad.py index bcceb4dea..b21bf7156 100644 --- a/examples/voice/voice-speechmatics-vad.py +++ b/examples/voice/voice-speechmatics-vad.py @@ -91,7 +91,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: stt = SpeechmaticsSTTService( - api_key=os.getenv("SPEECHMATICS_API_KEY"), + api_key=os.environ["SPEECHMATICS_API_KEY"], settings=SpeechmaticsSTTService.Settings( language=Language.EN, turn_detection_mode=SpeechmaticsSTTService.TurnDetectionMode.ADAPTIVE, @@ -102,7 +102,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = SpeechmaticsTTSService( - api_key=os.getenv("SPEECHMATICS_API_KEY"), + api_key=os.environ["SPEECHMATICS_API_KEY"], settings=SpeechmaticsTTSService.Settings( voice="sarah", ), @@ -110,7 +110,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( temperature=0.75, system_instruction="You are a helpful British assistant called Sarah in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Always include punctuation in your responses. Give very short replies - do not give longer replies unless strictly necessary. Respond to what the user said in a concise, funny, creative and helpful way. Use `` tags to identify different speakers - do not use tags in your replies. Do not respond to speakers within `` tags unless explicitly asked to.", diff --git a/examples/voice/voice-speechmatics.py b/examples/voice/voice-speechmatics.py index d619715aa..3f7fa9ac4 100644 --- a/examples/voice/voice-speechmatics.py +++ b/examples/voice/voice-speechmatics.py @@ -74,7 +74,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): async with aiohttp.ClientSession() as session: stt = SpeechmaticsSTTService( - api_key=os.getenv("SPEECHMATICS_API_KEY"), + api_key=os.environ["SPEECHMATICS_API_KEY"], settings=SpeechmaticsSTTService.Settings( language=Language.EN, speaker_active_format="<{speaker_id}>{text}", @@ -82,7 +82,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = SpeechmaticsTTSService( - api_key=os.getenv("SPEECHMATICS_API_KEY"), + api_key=os.environ["SPEECHMATICS_API_KEY"], settings=SpeechmaticsTTSService.Settings( voice="sarah", ), @@ -90,7 +90,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( temperature=0.75, system_instruction="You are a helpful British assistant called Sarah in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Always include punctuation in your responses. Give very short replies - do not give longer replies unless strictly necessary. Respond to what the user said in a concise, funny, creative and helpful way. Use `` tags to identify different speakers - do not use tags in your replies. Do not respond to speakers within `` tags unless explicitly asked to.", diff --git a/examples/voice/voice-xai.py b/examples/voice/voice-xai.py index e7fa04350..833f9b816 100644 --- a/examples/voice/voice-xai.py +++ b/examples/voice/voice-xai.py @@ -53,10 +53,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = XAISTTService(api_key=os.getenv("XAI_API_KEY")) + stt = XAISTTService(api_key=os.environ["XAI_API_KEY"]) tts = XAIHttpTTSService( - api_key=os.getenv("XAI_API_KEY"), + api_key=os.environ["XAI_API_KEY"], aiohttp_session=session, settings=XAIHttpTTSService.Settings( voice="eve", @@ -64,7 +64,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = GrokLLMService( - api_key=os.getenv("XAI_API_KEY"), + api_key=os.environ["XAI_API_KEY"], settings=GrokLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-xtts.py b/examples/voice/voice-xtts.py index f0bfe01c0..f9d92e45b 100644 --- a/examples/voice/voice-xtts.py +++ b/examples/voice/voice-xtts.py @@ -55,7 +55,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Create an HTTP session async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = XTTSService( aiohttp_session=session, @@ -66,7 +66,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/pyrightconfig.json b/pyrightconfig.json index 6dae89b0a..94c0114b6 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -13,10 +13,7 @@ "src/pipecat/pipeline", "src/pipecat/runner" ], - "exclude": [ - "**/*_pb2.py", - "**/__pycache__" - ], + "exclude": ["**/*_pb2.py", "**/__pycache__"], "ignore": [ "src/pipecat/adapters", "src/pipecat/audio", @@ -28,7 +25,6 @@ "src/pipecat/transports", "src/pipecat/utils", "tests", - "examples", "scripts", "docs" ], From d8f5c0be714d941e0f6fb0b684d37e8c2bbd4df1 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 20 Apr 2026 19:30:04 -0400 Subject: [PATCH 36/49] Add XAITTSService for xAI streaming WebSocket TTS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds XAITTSService in the existing xai/tts.py module, alongside the existing XAIHttpTTSService. Connects to xAI's streaming endpoint at wss://api.x.ai/v1/tts, streams text.delta chunks up and base64 audio.delta chunks down on the same connection so audio starts flowing before the full utterance is synthesized. Extends InterruptibleTTSService since xAI's protocol is strictly sequential per connection and exposes neither a cancel verb nor a context ID — the only way to stop an in-flight utterance is to tear down the WebSocket, which is exactly what InterruptibleTTSService does on interruption when the bot is speaking. Voice, language, codec, and sample_rate are passed as query-string params at connect time; runtime setting changes reconnect the socket. Defaults to raw PCM so emitted TTSAudioRawFrame objects need no decoding downstream. Splits the existing example into voice-xai.py (WebSocket) and voice-xai-http.py (batch HTTP) so each variant has its own entry point. Promotes the xai extra to depend on pipecat-ai[websockets-base] since the new service imports the websockets library. --- examples/voice/voice-xai-http.py | 128 ++++++++++++++ examples/voice/voice-xai.py | 6 +- src/pipecat/services/xai/tts.py | 280 ++++++++++++++++++++++++++++++- tests/test_xai_tts.py | 91 +++++++++- 4 files changed, 496 insertions(+), 9 deletions(-) create mode 100644 examples/voice/voice-xai-http.py diff --git a/examples/voice/voice-xai-http.py b/examples/voice/voice-xai-http.py new file mode 100644 index 000000000..6d0d54e79 --- /dev/null +++ b/examples/voice/voice-xai-http.py @@ -0,0 +1,128 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os + +import aiohttp +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.xai.llm import GrokLLMService +from pipecat.services.xai.tts import XAIHttpTTSService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +# We use lambdas to defer transport parameter creation until the transport +# type is selected at runtime. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + async with aiohttp.ClientSession() as session: + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = XAIHttpTTSService( + api_key=os.getenv("XAI_API_KEY"), + aiohttp_session=session, + settings=XAIHttpTTSService.Settings( + voice="eve", + ), + ) + + llm = GrokLLMService( + api_key=os.getenv("XAI_API_KEY"), + settings=GrokLLMService.Settings( + system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", + ), + ) + + context = LLMContext() + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + user_aggregator, # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + assistant_aggregator, # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + context.add_message( + {"role": "developer", "content": "Please introduce yourself to the user."} + ) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/voice/voice-xai.py b/examples/voice/voice-xai.py index 833f9b816..541d66c77 100644 --- a/examples/voice/voice-xai.py +++ b/examples/voice/voice-xai.py @@ -24,7 +24,7 @@ from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.xai.llm import GrokLLMService from pipecat.services.xai.stt import XAISTTService -from pipecat.services.xai.tts import XAIHttpTTSService +from pipecat.services.xai.tts import XAITTSService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams @@ -55,10 +55,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): async with aiohttp.ClientSession() as session: stt = XAISTTService(api_key=os.environ["XAI_API_KEY"]) - tts = XAIHttpTTSService( + tts = XAITTSService( api_key=os.environ["XAI_API_KEY"], aiohttp_session=session, - settings=XAIHttpTTSService.Settings( + settings=XAITTSService.Settings( voice="eve", ), ) diff --git a/src/pipecat/services/xai/tts.py b/src/pipecat/services/xai/tts.py index 17f67cf9a..f99772baa 100644 --- a/src/pipecat/services/xai/tts.py +++ b/src/pipecat/services/xai/tts.py @@ -6,22 +6,48 @@ """xAI text-to-speech service implementation. -Uses xAI's HTTP TTS endpoint documented at: -https://docs.x.ai/developers/model-capabilities/audio/text-to-speech +Provides two TTS services against xAI's voice API: + +- :class:`XAIHttpTTSService` uses the batch HTTP endpoint at + ``https://api.x.ai/v1/tts``. +- :class:`XAITTSService` uses the streaming WebSocket endpoint at + ``wss://api.x.ai/v1/tts``. + +See https://docs.x.ai/developers/rest-api-reference/inference/voice. """ +import base64 +import json from collections.abc import AsyncGenerator from dataclasses import dataclass +from typing import Any +from urllib.parse import urlencode import aiohttp from loguru import logger -from pipecat.frames.frames import ErrorFrame, Frame, TTSAudioRawFrame +from pipecat.frames.frames import ( + CancelFrame, + EndFrame, + ErrorFrame, + Frame, + StartFrame, + TTSAudioRawFrame, + TTSStoppedFrame, +) from pipecat.services.settings import TTSSettings -from pipecat.services.tts_service import TTSService +from pipecat.services.tts_service import InterruptibleTTSService, TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts +try: + from websockets.asyncio.client import connect as websocket_connect + from websockets.protocol import State +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use XAITTSService, you need to `pip install pipecat-ai[xai]`.") + raise Exception(f"Missing module: {e}") + def language_to_xai_language(language: Language) -> str | None: """Convert a Language enum to xAI language code. @@ -214,3 +240,249 @@ class XAIHttpTTSService(TTSService): ) except Exception as e: yield ErrorFrame(error=f"Unknown error occurred: {e}") + + +@dataclass +class XAIWebsocketTTSSettings(TTSSettings): + """Settings for XAITTSService (WebSocket streaming).""" + + pass + + +class XAITTSService(InterruptibleTTSService): + """xAI streaming text-to-speech service. + + Connects to xAI's WebSocket TTS endpoint and streams audio chunks back as + they are synthesized. Text can be sent incrementally via ``text.delta`` + messages and each utterance is terminated with ``text.done``. The server + responds with ``audio.delta`` chunks followed by an ``audio.done`` message. + + Audio parameters (voice, language, codec, sample rate, bit rate) are passed + as query string parameters on the WebSocket URL; changing any of them at + runtime reconnects the WebSocket. + """ + + Settings = XAIWebsocketTTSSettings + _settings: Settings + + def __init__( + self, + *, + api_key: str, + base_url: str = "wss://api.x.ai/v1/tts", + sample_rate: int | None = None, + codec: str = "pcm", + settings: Settings | None = None, + **kwargs, + ): + """Initialize the xAI WebSocket TTS service. + + Args: + api_key: xAI API key for authentication. + base_url: xAI TTS WebSocket endpoint. Defaults to + ``wss://api.x.ai/v1/tts``. + sample_rate: Output audio sample rate in Hz. If None, uses the + pipeline default. + codec: Output audio codec. One of ``pcm``, ``wav``, ``mulaw``, + ``alaw``. Defaults to ``pcm`` so emitted ``TTSAudioRawFrame`` + objects need no decoding downstream. + settings: Runtime-updatable settings. + **kwargs: Additional arguments passed to parent + ``InterruptibleTTSService``. + """ + default_settings = self.Settings( + model=None, + voice="eve", + language=Language.EN, + ) + + if settings is not None: + default_settings.apply_update(settings) + + super().__init__( + push_start_frame=True, + push_stop_frames=True, + sample_rate=sample_rate, + settings=default_settings, + **kwargs, + ) + + self._api_key = api_key + self._base_url = base_url + self._codec = codec + self._receive_task = None + + def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics.""" + return True + + def language_to_service_language(self, language: Language) -> str | None: + """Convert a Language enum to xAI language format.""" + return language_to_xai_language(language) + + async def start(self, frame: StartFrame): + """Start the xAI WebSocket TTS service.""" + await super().start(frame) + await self._connect() + + async def stop(self, frame: EndFrame): + """Stop the xAI WebSocket TTS service.""" + await super().stop(frame) + await self._disconnect() + + async def cancel(self, frame: CancelFrame): + """Cancel the xAI WebSocket TTS service.""" + await super().cancel(frame) + await self._disconnect() + + async def _connect(self): + await super()._connect() + + await self._connect_websocket() + + if self._websocket and not self._receive_task: + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) + + async def _disconnect(self): + await super()._disconnect() + + if self._receive_task: + await self.cancel_task(self._receive_task) + self._receive_task = None + + await self._disconnect_websocket() + + async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]: + """Apply a settings delta. Reconnects if any URL-baked field changes.""" + changed = await super()._update_settings(delta) + + if changed: + await self._disconnect() + await self._connect() + + return changed + + def _build_url(self) -> str: + language = self._settings.language + if isinstance(language, Language): + language_value = language_to_xai_language(language) or language.value + else: + language_value = str(language) if language is not None else "auto" + + params: dict[str, Any] = { + "voice": self._settings.voice, + "language": language_value, + "codec": self._codec, + "sample_rate": self.sample_rate, + } + return f"{self._base_url}?{urlencode(params)}" + + async def _connect_websocket(self): + try: + if self._websocket and self._websocket.state is State.OPEN: + return + + logger.debug("Connecting to xAI TTS") + + url = self._build_url() + headers = {"Authorization": f"Bearer {self._api_key}"} + self._websocket = await websocket_connect(url, additional_headers=headers) + + await self._call_event_handler("on_connected") + except Exception as e: + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) + self._websocket = None + await self._call_event_handler("on_connection_error", f"{e}") + + async def _disconnect_websocket(self): + try: + await self.stop_all_metrics() + + if self._websocket: + logger.debug("Disconnecting from xAI TTS") + await self._websocket.close() + except Exception as e: + await self.push_error(error_msg=f"Error disconnecting from xAI TTS: {e}", exception=e) + finally: + self._websocket = None + await self._call_event_handler("on_disconnected") + + def _get_websocket(self): + if self._websocket: + return self._websocket + raise Exception("Websocket not connected") + + async def flush_audio(self, context_id: str | None = None): + """Signal end-of-utterance so xAI begins synthesizing what it has buffered.""" + if not self._websocket or self._websocket.state is State.CLOSED: + return + await self._get_websocket().send(json.dumps({"type": "text.done"})) + + async def _receive_messages(self): + async for message in self._get_websocket(): + if isinstance(message, bytes): + logger.warning(f"{self}: unexpected binary frame from xAI TTS") + continue + try: + msg = json.loads(message) + except json.JSONDecodeError: + logger.error(f"{self}: invalid JSON message: {message}") + continue + + msg_type = msg.get("type") + context_id = self.get_active_audio_context_id() + + if msg_type == "audio.delta": + audio_b64 = msg.get("delta") + if not audio_b64: + continue + audio = base64.b64decode(audio_b64) + await self.stop_ttfb_metrics() + if context_id: + frame = TTSAudioRawFrame( + audio=audio, + sample_rate=self.sample_rate, + num_channels=1, + context_id=context_id, + ) + await self.append_to_audio_context(context_id, frame) + elif msg_type == "audio.done": + await self.stop_all_metrics() + if context_id: + await self.append_to_audio_context( + context_id, TTSStoppedFrame(context_id=context_id) + ) + await self.remove_audio_context(context_id) + elif msg_type == "error": + await self.stop_all_metrics() + error_detail = msg.get("message") or msg.get("error") or str(msg) + if context_id: + await self.append_to_audio_context( + context_id, TTSStoppedFrame(context_id=context_id) + ) + await self.remove_audio_context(context_id) + await self.push_error(error_msg=f"xAI TTS error: {error_detail}") + else: + logger.debug(f"{self}: unhandled xAI message type: {msg_type}") + + @traced_tts + async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]: + """Generate TTS audio from text using xAI's streaming WebSocket API.""" + logger.debug(f"{self}: Generating TTS [{text}]") + + try: + if not self._websocket or self._websocket.state is State.CLOSED: + await self._connect() + + try: + await self._get_websocket().send(json.dumps({"type": "text.delta", "delta": text})) + await self.start_tts_usage_metrics(text) + except Exception as e: + yield ErrorFrame(error=f"Unknown error occurred: {e}") + yield TTSStoppedFrame(context_id=context_id) + await self._disconnect() + await self._connect() + return + yield None + except Exception as e: + yield ErrorFrame(error=f"Unknown error occurred: {e}") diff --git a/tests/test_xai_tts.py b/tests/test_xai_tts.py index aab984567..98ac48fd4 100644 --- a/tests/test_xai_tts.py +++ b/tests/test_xai_tts.py @@ -4,14 +4,19 @@ # SPDX-License-Identifier: BSD 2-Clause License # -"""Tests for XAIHttpTTSService.""" +"""Tests for XAIHttpTTSService and XAITTSService.""" import asyncio +import base64 +import json import unittest +from urllib.parse import parse_qs, urlparse import aiohttp import pytest +import websockets from aiohttp import web +from websockets.asyncio.server import serve from pipecat.frames.frames import ( AggregatedTextFrame, @@ -21,7 +26,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, TTSTextFrame, ) -from pipecat.services.xai.tts import XAIHttpTTSService +from pipecat.services.xai.tts import XAIHttpTTSService, XAITTSService from pipecat.tests.utils import run_test @@ -87,5 +92,87 @@ async def test_run_xai_tts_success(aiohttp_client): } +@pytest.mark.asyncio +async def test_run_xai_websocket_tts_success(): + """xAI WS TTS should send text.delta+text.done and emit frames from audio.delta+audio.done.""" + + captured: dict = { + "request_path": None, + "auth_header": None, + "messages": [], + } + + audio_bytes = b"\x00\x01\x02\x03" * 1024 + + async def handler(ws): + request = ws.request + captured["request_path"] = request.path + captured["auth_header"] = request.headers.get("Authorization") + + try: + async for raw in ws: + msg = json.loads(raw) + captured["messages"].append(msg) + if msg.get("type") == "text.done": + await ws.send( + json.dumps( + { + "type": "audio.delta", + "delta": base64.b64encode(audio_bytes).decode("ascii"), + } + ) + ) + await ws.send(json.dumps({"type": "audio.done", "trace_id": "test-trace"})) + except websockets.ConnectionClosed: + pass + + async with serve(handler, "127.0.0.1", 0) as server: + host, port = next(iter(server.sockets)).getsockname()[:2] + base_url = f"ws://{host}:{port}/v1/tts" + + tts_service = XAITTSService( + api_key="test-key", + base_url=base_url, + sample_rate=24000, + ) + + down_frames, _ = await run_test( + tts_service, + frames_to_send=[TTSSpeakFrame(text="Hello from xAI."), _SleepAfterSpeak(0.3)], + ) + + frame_types = [type(frame) for frame in down_frames] + assert TTSStartedFrame in frame_types + assert TTSAudioRawFrame in frame_types + assert TTSStoppedFrame in frame_types + + audio_frames = [frame for frame in down_frames if isinstance(frame, TTSAudioRawFrame)] + assert audio_frames + assert all(frame.sample_rate == 24000 for frame in audio_frames) + assert all(frame.num_channels == 1 for frame in audio_frames) + assert b"".join(f.audio for f in audio_frames) == audio_bytes + + assert captured["auth_header"] == "Bearer test-key" + parsed = urlparse(captured["request_path"]) + query = parse_qs(parsed.query) + assert query["voice"] == ["eve"] + assert query["language"] == ["en"] + assert query["codec"] == ["pcm"] + assert query["sample_rate"] == ["24000"] + + types_sent = [m.get("type") for m in captured["messages"]] + assert "text.delta" in types_sent + assert "text.done" in types_sent + delta_msg = next(m for m in captured["messages"] if m.get("type") == "text.delta") + assert delta_msg["delta"] == "Hello from xAI." + + +# Small helper imported lazily to avoid circular import in fixture-lite tests. +def _SleepAfterSpeak(duration: float): + from pipecat.tests.utils import SleepFrame + + return SleepFrame(sleep=duration) + + if __name__ == "__main__": unittest.main() From 9a4951760987fe5a4f24f83d503c4d7024215bfb Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 20 Apr 2026 19:30:58 -0400 Subject: [PATCH 37/49] Add changelog entry for #4341 --- changelog/4341.added.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/4341.added.md diff --git a/changelog/4341.added.md b/changelog/4341.added.md new file mode 100644 index 000000000..9301bcc5f --- /dev/null +++ b/changelog/4341.added.md @@ -0,0 +1 @@ +- Added `XAITTSService` for streaming text-to-speech using xAI's WebSocket TTS endpoint (`wss://api.x.ai/v1/tts`). Streams `text.delta` chunks up and base64 `audio.delta` chunks down on the same connection so audio begins flowing before the full utterance finishes synthesizing; complements the batch-HTTP `XAIHttpTTSService`. Defaults to raw PCM output so `TTSAudioRawFrame` needs no decoding. The `xai` optional extra now pulls in `pipecat-ai[websockets-base]`. From 84891de04d7c22ce8093f0419043fc1c82249b24 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 21 Apr 2026 15:49:59 -0400 Subject: [PATCH 38/49] Add voice/xai-http.py to release evals --- scripts/evals/run-release-evals.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index ebf5b36c2..861367027 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -108,6 +108,7 @@ TESTS_VOICE = [ ("voice/voice-elevenlabs.py", EVAL_SIMPLE_MATH), ("voice/voice-elevenlabs-http.py", EVAL_SIMPLE_MATH), ("voice/voice-xai.py", EVAL_SIMPLE_MATH), + ("voice/voice-xai-http.py", EVAL_SIMPLE_MATH), ("voice/voice-azure.py", EVAL_SIMPLE_MATH), ("voice/voice-azure-http.py", EVAL_SIMPLE_MATH), ("voice/voice-openai.py", EVAL_SIMPLE_MATH), From 10e58d6e42794bae18f18431c9b3915d59609f8c Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 21 Apr 2026 16:17:49 -0400 Subject: [PATCH 39/49] Fix type errors in scripts and add to pyright checked set --- pyrightconfig.json | 7 +++---- scripts/daily/test_tavus_transport.py | 18 +++++++++++------- scripts/evals/eval.py | 10 +++++----- scripts/evals/run-eval.py | 5 +++-- scripts/evals/utils.py | 2 ++ .../krisp/test_krisp_viva_turn_audiofile.py | 2 +- 6 files changed, 25 insertions(+), 19 deletions(-) diff --git a/pyrightconfig.json b/pyrightconfig.json index 94c0114b6..e453d8c60 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -11,7 +11,8 @@ "src/pipecat/extensions", "src/pipecat/turns", "src/pipecat/pipeline", - "src/pipecat/runner" + "src/pipecat/runner", + "scripts" ], "exclude": ["**/*_pb2.py", "**/__pycache__"], "ignore": [ @@ -24,9 +25,7 @@ "src/pipecat/tests", "src/pipecat/transports", "src/pipecat/utils", - "tests", - "scripts", - "docs" + "tests" ], "reportMissingImports": false } diff --git a/scripts/daily/test_tavus_transport.py b/scripts/daily/test_tavus_transport.py index fa8afe835..ffb6b8ee6 100644 --- a/scripts/daily/test_tavus_transport.py +++ b/scripts/daily/test_tavus_transport.py @@ -2,7 +2,14 @@ import asyncio import os import signal -from daily import * +from daily import ( + AudioData, + CallClient, + CustomAudioSource, + CustomAudioTrack, + Daily, + EventHandler, +) from dotenv import load_dotenv from loguru import logger @@ -33,8 +40,8 @@ class DailyProxyApp(EventHandler): def __init__(self, sample_rate: int): super().__init__() self._sample_rate = sample_rate - self._loop = None - self._audio_queue: asyncio.Queue | None = None + self._loop = asyncio.new_event_loop() + self._audio_queue: asyncio.Queue = asyncio.Queue() self._audio_task: asyncio.Task | None = None self._client: CallClient = CallClient(event_handler=self) @@ -52,7 +59,6 @@ class DailyProxyApp(EventHandler): self._loop.call_soon_threadsafe(self._loop.stop) def run(self, meeting_url: str): - self._loop = asyncio.new_event_loop() asyncio.set_event_loop(self._loop) self._create_audio_task() @@ -101,7 +107,6 @@ class DailyProxyApp(EventHandler): def _create_audio_task(self): if not self._audio_task: - self._audio_queue = asyncio.Queue() self._audio_task = self._loop.create_task(self._audio_task_handler()) async def _cancel_audio_task(self): @@ -113,7 +118,6 @@ class DailyProxyApp(EventHandler): except asyncio.CancelledError: pass self._audio_task = None - self._audio_queue = None async def capture_participant_audio(self, participant_id: str): logger.info(f"Capturing participant audio: {participant_id}") @@ -168,7 +172,7 @@ class DailyProxyApp(EventHandler): def main(): Daily.init() - room_url = os.getenv("TAVUS_SAMPLE_ROOM_URL") + room_url = os.environ["TAVUS_SAMPLE_ROOM_URL"] app = DailyProxyApp(sample_rate=24000) app.run(room_url) diff --git a/scripts/evals/eval.py b/scripts/evals/eval.py index cf7ecf257..fdec8238c 100644 --- a/scripts/evals/eval.py +++ b/scripts/evals/eval.py @@ -198,7 +198,7 @@ class EvalRunner: async def run_example_pipeline(script_path: Path, eval_config: EvalConfig): - room_url = os.getenv("DAILY_ROOM_URL") + room_url = os.environ["DAILY_ROOM_URL"] module = load_module_from_path(script_path) @@ -227,7 +227,7 @@ async def run_eval_pipeline( ): logger.info(f"Starting eval bot") - room_url = os.getenv("DAILY_ROOM_URL") + room_url = os.environ["DAILY_ROOM_URL"] transport = DailyTransport( room_url, @@ -243,7 +243,7 @@ async def run_eval_pipeline( # We disable smart formatting because some times if the user says "3 + 2 is # 5" (in audio) this can be converted to "32 is 5". stt = DeepgramSTTService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], settings=DeepgramSTTService.Settings( language="multi", smart_format=False, @@ -251,7 +251,7 @@ async def run_eval_pipeline( ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="97f4b8fb-f2fe-444b-bb9a-c109783a857a", # Nathan ), @@ -375,7 +375,7 @@ async def run_eval_pipeline( @task.event_handler("on_pipeline_finished") async def on_pipeline_finished(task, frame): if isinstance(frame, EndFrame): - await eval_runner.assert_eval(frame.reason) + await eval_runner.assert_eval(bool(frame.reason)) elif isinstance(frame, CancelFrame): await eval_runner.assert_eval(False) diff --git a/scripts/evals/run-eval.py b/scripts/evals/run-eval.py index 72d4902b7..5ccb4f7c6 100644 --- a/scripts/evals/run-eval.py +++ b/scripts/evals/run-eval.py @@ -11,7 +11,7 @@ import sys from pathlib import Path from dotenv import load_dotenv -from eval import EvalRunner +from eval import EvalConfig, EvalRunner from loguru import logger from utils import check_env_variables @@ -33,7 +33,8 @@ async def main(args: argparse.Namespace): runner = EvalRunner(examples_dir=script_path, record_audio=args.audio, log_level=log_level) - await runner.run_eval(script_file, args.prompt, args.eval) + eval_config = EvalConfig(prompt=args.prompt, eval=args.eval) + await runner.run_eval(script_file, eval_config) runner.print_results() diff --git a/scripts/evals/utils.py b/scripts/evals/utils.py index 7ce9dfe84..997ef8f73 100644 --- a/scripts/evals/utils.py +++ b/scripts/evals/utils.py @@ -79,6 +79,8 @@ def load_module_from_path(path: str | Path): module_name = path.stem spec = importlib.util.spec_from_file_location(module_name, str(path)) + if spec is None or spec.loader is None: + raise ImportError(f"Could not load module spec from {path}") module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module diff --git a/scripts/krisp/test_krisp_viva_turn_audiofile.py b/scripts/krisp/test_krisp_viva_turn_audiofile.py index 6580e8e90..99557c171 100644 --- a/scripts/krisp/test_krisp_viva_turn_audiofile.py +++ b/scripts/krisp/test_krisp_viva_turn_audiofile.py @@ -71,7 +71,7 @@ async def analyze_audio_file( frame_duration_ms: int = 20, chunk_duration_ms: int = 20, verbose: bool = False, - output_file: str = None, + output_file: str | None = None, ) -> None: """Analyze an audio file for turn detection using Krisp VIVA turn analyzer. From 847bd8af4bd5521f98dd13d56c4de81eb11375f7 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 21 Apr 2026 16:21:46 -0400 Subject: [PATCH 40/49] Remove src/pipecat/sync which doesn't exist --- pyrightconfig.json | 1 - 1 file changed, 1 deletion(-) diff --git a/pyrightconfig.json b/pyrightconfig.json index e453d8c60..694e9ab8a 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -21,7 +21,6 @@ "src/pipecat/processors", "src/pipecat/serializers", "src/pipecat/services", - "src/pipecat/sync", "src/pipecat/tests", "src/pipecat/transports", "src/pipecat/utils", From c244a950eb82ccc2bd5c83ce9913d864f73bf6c3 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 21 Apr 2026 16:24:53 -0400 Subject: [PATCH 41/49] Add src/pipecat/tests to include list, alphabetize list --- pyrightconfig.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pyrightconfig.json b/pyrightconfig.json index 694e9ab8a..822ccbf22 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -3,16 +3,17 @@ "pythonVersion": "3.11", "pythonPlatform": "All", "include": [ + "scripts", "src/pipecat/clocks", - "src/pipecat/metrics", - "src/pipecat/transcriptions", - "src/pipecat/frames", - "src/pipecat/observers", "src/pipecat/extensions", - "src/pipecat/turns", + "src/pipecat/frames", + "src/pipecat/metrics", + "src/pipecat/observers", "src/pipecat/pipeline", "src/pipecat/runner", - "scripts" + "src/pipecat/tests", + "src/pipecat/transcriptions", + "src/pipecat/turns" ], "exclude": ["**/*_pb2.py", "**/__pycache__"], "ignore": [ @@ -21,7 +22,6 @@ "src/pipecat/processors", "src/pipecat/serializers", "src/pipecat/services", - "src/pipecat/tests", "src/pipecat/transports", "src/pipecat/utils", "tests" From 308044808dcfe63001612158d0416ec19db6d8f9 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 21 Apr 2026 16:39:30 -0400 Subject: [PATCH 42/49] Rename to _user_speech_wait_done --- .../speech_timeout_user_turn_stop_strategy.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/pipecat/turns/user_stop/speech_timeout_user_turn_stop_strategy.py b/src/pipecat/turns/user_stop/speech_timeout_user_turn_stop_strategy.py index 9a4060f1c..c27767cbb 100644 --- a/src/pipecat/turns/user_stop/speech_timeout_user_turn_stop_strategy.py +++ b/src/pipecat/turns/user_stop/speech_timeout_user_turn_stop_strategy.py @@ -66,7 +66,7 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): # VAD-driven timers and completion flags. self._user_speech_timeout_task: asyncio.Task | None = None self._stt_timeout_task: asyncio.Task | None = None - self._user_speech_expired: bool = False + self._user_speech_wait_done: bool = False self._stt_wait_done: bool = False # Fallback timer (transcript arrived without VAD stop). @@ -80,7 +80,7 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): self._vad_user_speaking = False self._transcript_finalized = False self._vad_stopped_time = None - self._user_speech_expired = False + self._user_speech_wait_done = False self._stt_wait_done = False self._fallback_expired = False await self._cancel_all_tasks() @@ -129,7 +129,7 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): self._vad_user_speaking = True self._transcript_finalized = False self._vad_stopped_time = None - self._user_speech_expired = False + self._user_speech_wait_done = False self._stt_wait_done = False self._fallback_expired = False await self._cancel_all_tasks() @@ -202,7 +202,7 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): # If both VAD-path timers are done (or the fallback timer already # expired), the turn was waiting on text — trigger now. - if self._fallback_expired or (self._user_speech_expired and self._stt_wait_done): + if self._fallback_expired or (self._user_speech_wait_done and self._stt_wait_done): await self._maybe_trigger_user_turn_stopped() return @@ -231,7 +231,7 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): finally: self._user_speech_timeout_task = None - self._user_speech_expired = True + self._user_speech_wait_done = True await self._maybe_trigger_user_turn_stopped() async def _stt_timeout_handler(self, timeout: float): @@ -280,7 +280,7 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): return if self._vad_stopped_time is not None: - if self._user_speech_expired and self._stt_wait_done: + if self._user_speech_wait_done and self._stt_wait_done: await self.trigger_user_turn_stopped() return From 21f5cfe21ad943007ad65730e316ada8d9b587d7 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 21 Apr 2026 16:47:12 -0400 Subject: [PATCH 43/49] Fix type errors in utils and add to pyright checked set --- pyrightconfig.json | 4 ++-- .../context/llm_context_summarization.py | 22 +++++++++++++------ src/pipecat/utils/frame_queue.py | 2 +- src/pipecat/utils/string.py | 2 +- .../utils/text/base_text_aggregator.py | 2 +- .../utils/text/pattern_pair_aggregator.py | 2 +- .../utils/tracing/service_attributes.py | 4 ++-- 7 files changed, 23 insertions(+), 15 deletions(-) diff --git a/pyrightconfig.json b/pyrightconfig.json index 822ccbf22..31bac33c8 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -13,7 +13,8 @@ "src/pipecat/runner", "src/pipecat/tests", "src/pipecat/transcriptions", - "src/pipecat/turns" + "src/pipecat/turns", + "src/pipecat/utils" ], "exclude": ["**/*_pb2.py", "**/__pycache__"], "ignore": [ @@ -23,7 +24,6 @@ "src/pipecat/serializers", "src/pipecat/services", "src/pipecat/transports", - "src/pipecat/utils", "tests" ], "reportMissingImports": false diff --git a/src/pipecat/utils/context/llm_context_summarization.py b/src/pipecat/utils/context/llm_context_summarization.py index 85a7e6e19..59de6f4fa 100644 --- a/src/pipecat/utils/context/llm_context_summarization.py +++ b/src/pipecat/utils/context/llm_context_summarization.py @@ -20,7 +20,11 @@ if TYPE_CHECKING: from loguru import logger -from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage +from pipecat.processors.aggregators.llm_context import ( + LLMContext, + LLMContextMessage, + LLMSpecificMessage, +) # Fallback timeout (seconds) used when summarization_timeout is None. DEFAULT_SUMMARIZATION_TIMEOUT = 120.0 @@ -269,7 +273,7 @@ class LLMMessagesToSummarize: last_summarized_index: Index of the last message being summarized """ - messages: list[dict] + messages: list[LLMContextMessage] last_summarized_index: int @@ -415,7 +419,7 @@ class LLMContextSummarizationUtil: @staticmethod def _get_earliest_function_call_not_resolved_in_range( - messages: list[dict], start_idx: int, summary_end: int + messages: list[LLMContextMessage], start_idx: int, summary_end: int ) -> int: """Find the earliest message index with incomplete function calls. @@ -470,9 +474,10 @@ class LLMContextSummarizationUtil: if role == "tool": tool_call_id = msg.get("tool_call_id") if tool_call_id and tool_call_id in pending_tool_calls: - if not LLMContextSummarizationUtil._is_tool_message_pending( - msg.get("content", "") - ): + content = msg.get("content", "") + if not isinstance(content, str): + content = "" + if not LLMContextSummarizationUtil._is_tool_message_pending(content): pending_tool_calls.pop(tool_call_id) # Check for async tool completion — a developer message with @@ -480,7 +485,10 @@ class LLMContextSummarizationUtil: # async result has arrived and the call is now resolved. if role == "developer": try: - parsed = json.loads(msg.get("content", "")) + content = msg.get("content", "") + if not isinstance(content, str): + continue + parsed = json.loads(content) if ( isinstance(parsed, dict) and parsed.get("type") == "async_tool" diff --git a/src/pipecat/utils/frame_queue.py b/src/pipecat/utils/frame_queue.py index 1b787362d..14347681a 100644 --- a/src/pipecat/utils/frame_queue.py +++ b/src/pipecat/utils/frame_queue.py @@ -58,7 +58,7 @@ class FrameQueue(asyncio.Queue): Returns: True if at least one enqueued frame is an instance of ``frame_type``. """ - for item in self._queue: + for item in self._queue: # pyright: ignore[reportAttributeAccessIssue] if isinstance(self._frame_getter(item), frame_type): return True return False diff --git a/src/pipecat/utils/string.py b/src/pipecat/utils/string.py index 55b1c2d53..c464e189f 100644 --- a/src/pipecat/utils/string.py +++ b/src/pipecat/utils/string.py @@ -234,7 +234,7 @@ class TextPartForConcatenation: includes_inter_part_spaces: bool def __str__(self): - return f"{self.name}(text: [{self.text}], includes_inter_part_spaces: {self.includes_inter_part_spaces})" + return f"{type(self).__name__}(text: [{self.text}], includes_inter_part_spaces: {self.includes_inter_part_spaces})" def concatenate_aggregated_text(text_parts: list[TextPartForConcatenation]) -> str: diff --git a/src/pipecat/utils/text/base_text_aggregator.py b/src/pipecat/utils/text/base_text_aggregator.py index 99ca145b6..b1b933c81 100644 --- a/src/pipecat/utils/text/base_text_aggregator.py +++ b/src/pipecat/utils/text/base_text_aggregator.py @@ -125,7 +125,7 @@ class BaseTextAggregator(ABC): """ pass # Make this a generator to satisfy type checker - yield # pragma: no cover + yield # pyright: ignore[reportReturnType] # pragma: no cover @abstractmethod async def flush(self) -> Aggregation | None: diff --git a/src/pipecat/utils/text/pattern_pair_aggregator.py b/src/pipecat/utils/text/pattern_pair_aggregator.py index 975413c78..67e044152 100644 --- a/src/pipecat/utils/text/pattern_pair_aggregator.py +++ b/src/pipecat/utils/text/pattern_pair_aggregator.py @@ -273,7 +273,7 @@ class PatternPairAggregator(SimpleTextAggregator): # Which is why we base the return on the first found. if start_count > end_count: start_index = text.find(start) - return [start_index, pattern_info] + return (start_index, pattern_info) return None diff --git a/src/pipecat/utils/tracing/service_attributes.py b/src/pipecat/utils/tracing/service_attributes.py index d9b86a9a4..a8b33e225 100644 --- a/src/pipecat/utils/tracing/service_attributes.py +++ b/src/pipecat/utils/tracing/service_attributes.py @@ -440,7 +440,7 @@ def add_openai_realtime_span_attributes( if isinstance(tool, dict) and "name" in tool: tool_names.append(tool["name"]) elif hasattr(tool, "name"): - tool_names.append(tool.name) + tool_names.append(getattr(tool, "name")) elif isinstance(tool, dict) and "function" in tool and "name" in tool["function"]: tool_names.append(tool["function"]["name"]) @@ -455,7 +455,7 @@ def add_openai_realtime_span_attributes( if function_calls: call = function_calls[0] if hasattr(call, "name"): - span.set_attribute("function_calls.first_name", call.name) + span.set_attribute("function_calls.first_name", getattr(call, "name")) elif isinstance(call, dict) and "name" in call: span.set_attribute("function_calls.first_name", call["name"]) From 4d9dc64af8067cba202618fa1e5cd2ecf8095b17 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 21 Apr 2026 16:51:27 -0400 Subject: [PATCH 44/49] Install all extras in format workflow for pyright CI was running `uv sync --group dev` without extras. Adds daily and tracing to extras. --- .github/workflows/format.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/format.yaml b/.github/workflows/format.yaml index c02245708..7ac009bdc 100644 --- a/.github/workflows/format.yaml +++ b/.github/workflows/format.yaml @@ -32,7 +32,7 @@ jobs: run: uv python install 3.12 - name: Install development dependencies - run: uv sync --group dev + run: uv sync --group dev --extra daily --extra tracing - name: Ruff formatter id: ruff-format From dcbe86d0fc9c14f033edaef977c20d8cf3650712 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 21 Apr 2026 17:33:12 -0400 Subject: [PATCH 45/49] Unify fallback timeout into the user-speech timer Collapse the separate fallback timer into the existing user_speech_timeout timer, restarted when a transcript arrives without a VAD stop. stt_timeout has no meaning on the fallback path, so the stt wait is marked done immediately. This drops the _fallback_timeout_task / _fallback_expired bookkeeping and the branched trigger condition. --- .../speech_timeout_user_turn_stop_strategy.py | 101 +++++++----------- 1 file changed, 36 insertions(+), 65 deletions(-) diff --git a/src/pipecat/turns/user_stop/speech_timeout_user_turn_stop_strategy.py b/src/pipecat/turns/user_stop/speech_timeout_user_turn_stop_strategy.py index c27767cbb..254fa60b1 100644 --- a/src/pipecat/turns/user_stop/speech_timeout_user_turn_stop_strategy.py +++ b/src/pipecat/turns/user_stop/speech_timeout_user_turn_stop_strategy.py @@ -39,9 +39,10 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): means STT has nothing more to send. Fallback: when a transcript arrives without a VAD stop event, the - strategy waits only user_speech_timeout for inactivity (rearmed on each - transcript). stt_timeout has no meaning here since it is defined - relative to VAD stop, and STT has already emitted a transcript. + user_speech_timeout timer measures inactivity since the last transcript + (rearmed on each transcript). stt_timeout has no meaning here since it + is defined relative to VAD stop, and STT has already emitted a + transcript — so the stt wait is marked done immediately. """ def __init__(self, *, user_speech_timeout: float = 0.6, **kwargs): @@ -63,16 +64,11 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): self._transcript_finalized = False self._vad_stopped_time: float | None = None - # VAD-driven timers and completion flags. self._user_speech_timeout_task: asyncio.Task | None = None self._stt_timeout_task: asyncio.Task | None = None self._user_speech_wait_done: bool = False self._stt_wait_done: bool = False - # Fallback timer (transcript arrived without VAD stop). - self._fallback_timeout_task: asyncio.Task | None = None - self._fallback_expired: bool = False - async def reset(self): """Reset the strategy to its initial state.""" await super().reset() @@ -82,7 +78,6 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): self._vad_stopped_time = None self._user_speech_wait_done = False self._stt_wait_done = False - self._fallback_expired = False await self._cancel_all_tasks() async def setup(self, task_manager: BaseTaskManager): @@ -131,7 +126,6 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): self._vad_stopped_time = None self._user_speech_wait_done = False self._stt_wait_done = False - self._fallback_expired = False await self._cancel_all_tasks() async def _handle_vad_user_stopped_speaking(self, frame: VADUserStoppedSpeakingFrame): @@ -160,20 +154,13 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): f"user_turn_stop_timeout parameter in the LLMUserAggregatorParams." ) - # Any prior fallback timer is superseded by the VAD-driven path. - if self._fallback_timeout_task: - await self.task_manager.cancel_task(self._fallback_timeout_task) - self._fallback_timeout_task = None - self._fallback_expired = False - - # user_speech_timeout is the policy floor and always runs. - self._user_speech_timeout_task = self.task_manager.create_task( - self._user_speech_timeout_handler(self._user_speech_timeout), - f"{self}::_user_speech_timeout_handler", - ) + # user_speech_timeout is the policy floor and always runs. A prior + # fallback-mode run of the same timer is superseded here. + await self._restart_user_speech_timer() # stt_timeout is a safety net. Short-circuit it if the transcript is # already finalized, or if the VAD stop_secs already covered it. + self._stt_wait_done = False effective_stt_wait = max(0.0, self._stt_timeout - self._stop_secs) if self._transcript_finalized or effective_stt_wait <= 0: self._stt_wait_done = True @@ -200,23 +187,33 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): await self.task_manager.cancel_task(self._stt_timeout_task) self._stt_timeout_task = None - # If both VAD-path timers are done (or the fallback timer already - # expired), the turn was waiting on text — trigger now. - if self._fallback_expired or (self._user_speech_wait_done and self._stt_wait_done): + # If both waits are already done, the turn was waiting on text — + # trigger now. + if self._user_speech_wait_done and self._stt_wait_done: await self._maybe_trigger_user_turn_stopped() return - # Fallback: handle transcripts when no VAD stop was received. - # Rearm the fallback timer on each transcript to wait for inactivity. + # Fallback: transcript arrived without a VAD stop. Measure inactivity + # since the last transcript with the user_speech_timer. stt_timeout + # has no meaning here (it's defined relative to VAD stop), so mark + # the stt wait done immediately. if not self._vad_user_speaking and self._vad_stopped_time is None: - if self._fallback_timeout_task: - await self.task_manager.cancel_task(self._fallback_timeout_task) - self._fallback_timeout_task = self.task_manager.create_task( - self._fallback_timeout_handler(self._user_speech_timeout), - f"{self}::_fallback_timeout_handler", - ) - # Make sure the task is scheduled. - await asyncio.sleep(0) + self._stt_wait_done = True + await self._restart_user_speech_timer() + + async def _restart_user_speech_timer(self): + """Cancel any running user_speech timer and start a fresh one.""" + if self._user_speech_timeout_task: + await self.task_manager.cancel_task(self._user_speech_timeout_task) + self._user_speech_timeout_task = None + self._user_speech_wait_done = False + self._user_speech_timeout_task = self.task_manager.create_task( + self._user_speech_timeout_handler(self._user_speech_timeout), + f"{self}::_user_speech_timeout_handler", + ) + # Make sure the task is scheduled so it can't be cancelled before + # starting (which would leave its coroutine un-awaited). + await asyncio.sleep(0) async def _user_speech_timeout_handler(self, timeout: float): """Wait user_speech_timeout then attempt to trigger user turn stopped. @@ -250,41 +247,18 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): self._stt_wait_done = True await self._maybe_trigger_user_turn_stopped() - async def _fallback_timeout_handler(self, timeout: float): - """Wait user_speech_timeout of inactivity on the fallback path. - - Args: - timeout: The timeout in seconds to wait. - """ - try: - await asyncio.sleep(timeout) - except asyncio.CancelledError: - return - finally: - self._fallback_timeout_task = None - - self._fallback_expired = True - await self._maybe_trigger_user_turn_stopped() - async def _maybe_trigger_user_turn_stopped(self): """Trigger user turn stopped if all required conditions are met. - VAD path: both user_speech_timeout and stt_timeout must have - completed (stt short-circuited by finalization counts as complete). - Fallback path: the fallback timer must have completed. - - In all cases, the user must not be currently speaking and at least - one transcript must have been received. + Both timers must be done (stt is marked done immediately on the + fallback path and when finalization short-circuits the safety net), + the user must not be currently speaking, and at least one transcript + must have been received. """ if self._vad_user_speaking or not self._text: return - if self._vad_stopped_time is not None: - if self._user_speech_wait_done and self._stt_wait_done: - await self.trigger_user_turn_stopped() - return - - if self._fallback_expired: + if self._user_speech_wait_done and self._stt_wait_done: await self.trigger_user_turn_stopped() async def _cancel_all_tasks(self): @@ -295,6 +269,3 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): if self._stt_timeout_task: await self.task_manager.cancel_task(self._stt_timeout_task) self._stt_timeout_task = None - if self._fallback_timeout_task: - await self.task_manager.cancel_task(self._fallback_timeout_task) - self._fallback_timeout_task = None From df9642eb5afb0ef7a42d06ab8f59d89bd6a1d56f Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 21 Apr 2026 18:12:54 -0400 Subject: [PATCH 46/49] Fix type errors in serializers and add to pyright checked set Moves src/pipecat/serializers into pyright's include list. Narrows self._params to each subclass's InputParams in exotel, vonage, plivo, twilio, genesys, and telnyx. In protobuf.py, renames the reassigned frame local to avoid clobbering its Frame type and silences two dynamic attribute accesses on the generated frames_pb2 module. Also aligns telnyx and plivo hangup validation with twilio: if auto_hang_up=True (the default) but required credentials are missing, __init__ now raises ValueError instead of silently logging a warning at call-end time. Previously a misconfigured serializer would construct fine and fail to hang up the call later, leaving a phantom billable session. --- pyrightconfig.json | 2 +- src/pipecat/serializers/exotel.py | 4 ++- src/pipecat/serializers/genesys.py | 4 ++- src/pipecat/serializers/plivo.py | 42 ++++++++++++++++------------- src/pipecat/serializers/protobuf.py | 17 ++++++------ src/pipecat/serializers/telnyx.py | 34 +++++++++++++++-------- src/pipecat/serializers/twilio.py | 13 ++++++--- src/pipecat/serializers/vonage.py | 4 ++- 8 files changed, 75 insertions(+), 45 deletions(-) diff --git a/pyrightconfig.json b/pyrightconfig.json index 31bac33c8..cd1a69be2 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -11,6 +11,7 @@ "src/pipecat/observers", "src/pipecat/pipeline", "src/pipecat/runner", + "src/pipecat/serializers", "src/pipecat/tests", "src/pipecat/transcriptions", "src/pipecat/turns", @@ -21,7 +22,6 @@ "src/pipecat/adapters", "src/pipecat/audio", "src/pipecat/processors", - "src/pipecat/serializers", "src/pipecat/services", "src/pipecat/transports", "tests" diff --git a/src/pipecat/serializers/exotel.py b/src/pipecat/serializers/exotel.py index ff2510f57..90d28270a 100644 --- a/src/pipecat/serializers/exotel.py +++ b/src/pipecat/serializers/exotel.py @@ -59,7 +59,9 @@ class ExotelFrameSerializer(FrameSerializer): call_sid: The associated Exotel Call SID (optional, not used in this implementation). params: Configuration parameters. """ - super().__init__(params or ExotelFrameSerializer.InputParams()) + params = params or ExotelFrameSerializer.InputParams() + super().__init__(params) + self._params: ExotelFrameSerializer.InputParams = params self._stream_sid = stream_sid self._call_sid = call_sid diff --git a/src/pipecat/serializers/genesys.py b/src/pipecat/serializers/genesys.py index e52abc6aa..9fca1f89a 100644 --- a/src/pipecat/serializers/genesys.py +++ b/src/pipecat/serializers/genesys.py @@ -166,7 +166,9 @@ class GenesysAudioHookSerializer(FrameSerializer): params: Configuration parameters. **kwargs: Additional arguments passed to BaseObject (e.g., name). """ - super().__init__(params or GenesysAudioHookSerializer.InputParams(), **kwargs) + params = params or GenesysAudioHookSerializer.InputParams() + super().__init__(params, **kwargs) + self._params: GenesysAudioHookSerializer.InputParams = params self._genesys_sample_rate = self._params.genesys_sample_rate self._sample_rate = 0 # Pipeline input rate, set in setup() diff --git a/src/pipecat/serializers/plivo.py b/src/pipecat/serializers/plivo.py index e86d18b8c..0df2d6753 100644 --- a/src/pipecat/serializers/plivo.py +++ b/src/pipecat/serializers/plivo.py @@ -8,6 +8,7 @@ import base64 import json +from typing import cast from loguru import logger @@ -71,7 +72,24 @@ class PlivoFrameSerializer(FrameSerializer): auth_token: Plivo auth token (required for auto hang-up). params: Configuration parameters. """ - super().__init__(params or PlivoFrameSerializer.InputParams()) + params = params or PlivoFrameSerializer.InputParams() + super().__init__(params) + self._params: PlivoFrameSerializer.InputParams = params + + # Validate hangup-related parameters if auto_hang_up is enabled + if self._params.auto_hang_up: + missing_credentials = [] + if not call_id: + missing_credentials.append("call_id") + if not auth_id: + missing_credentials.append("auth_id") + if not auth_token: + missing_credentials.append("auth_token") + + if missing_credentials: + raise ValueError( + f"auto_hang_up is enabled but missing required parameters: {', '.join(missing_credentials)}" + ) self._stream_id = stream_id self._call_id = call_id @@ -152,23 +170,11 @@ class PlivoFrameSerializer(FrameSerializer): try: import aiohttp - auth_id = self._auth_id - auth_token = self._auth_token - call_id = self._call_id - - if not call_id or not auth_id or not auth_token: - missing = [] - if not call_id: - missing.append("call_id") - if not auth_id: - missing.append("auth_id") - if not auth_token: - missing.append("auth_token") - - logger.warning( - f"Cannot hang up Plivo call: missing required parameters: {', '.join(missing)}" - ) - return + # __init__ guarantees these are non-None whenever auto_hang_up is True, + # which is the only path that reaches this method. + auth_id = cast(str, self._auth_id) + auth_token = cast(str, self._auth_token) + call_id = cast(str, self._call_id) # Plivo API endpoint for hanging up calls endpoint = f"https://api.plivo.com/v1/Account/{auth_id}/Call/{call_id}/" diff --git a/src/pipecat/serializers/protobuf.py b/src/pipecat/serializers/protobuf.py index 78d20fa24..44c660042 100644 --- a/src/pipecat/serializers/protobuf.py +++ b/src/pipecat/serializers/protobuf.py @@ -83,23 +83,24 @@ class ProtobufFrameSerializer(FrameSerializer): Serialized frame as bytes, or None if frame type is not serializable. """ # Wrapping this messages as a JSONFrame to send + serializable: Frame | MessageFrame = frame if isinstance(frame, (OutputTransportMessageFrame, OutputTransportMessageUrgentFrame)): if self.should_ignore_frame(frame): return None - frame = MessageFrame( + serializable = MessageFrame( data=json.dumps(frame.message), ) - proto_frame = frame_protos.Frame() - if type(frame) not in self.SERIALIZABLE_TYPES: - logger.warning(f"Frame type {type(frame)} is not serializable") + proto_frame = frame_protos.Frame() # type: ignore[attr-defined] + if type(serializable) not in self.SERIALIZABLE_TYPES: + logger.warning(f"Frame type {type(serializable)} is not serializable") return None # ignoring linter errors; we check that type(frame) is in this dict above - proto_optional_name = self.SERIALIZABLE_TYPES[type(frame)] # type: ignore + proto_optional_name = self.SERIALIZABLE_TYPES[type(serializable)] # type: ignore proto_attr = getattr(proto_frame, proto_optional_name) - for field in dataclasses.fields(frame): # type: ignore - value = getattr(frame, field.name) + for field in dataclasses.fields(serializable): # type: ignore + value = getattr(serializable, field.name) if value and hasattr(proto_attr, field.name): setattr(proto_attr, field.name, value) @@ -114,7 +115,7 @@ class ProtobufFrameSerializer(FrameSerializer): Returns: Deserialized frame instance, or None if deserialization fails. """ - proto = frame_protos.Frame.FromString(data) + proto = frame_protos.Frame.FromString(data) # type: ignore[attr-defined] which = proto.WhichOneof("frame") if which not in self.DESERIALIZABLE_FIELDS: logger.error("Unable to deserialize a valid frame") diff --git a/src/pipecat/serializers/telnyx.py b/src/pipecat/serializers/telnyx.py index 0d74664ab..4d769cada 100644 --- a/src/pipecat/serializers/telnyx.py +++ b/src/pipecat/serializers/telnyx.py @@ -8,10 +8,10 @@ import base64 import json +from typing import cast import aiohttp from loguru import logger -from pydantic import BaseModel from pipecat.audio.dtmf.types import KeypadEntry from pipecat.audio.utils import ( @@ -46,7 +46,7 @@ class TelnyxFrameSerializer(FrameSerializer): credentials to be provided. """ - class InputParams(BaseModel): + class InputParams(FrameSerializer.InputParams): """Configuration parameters for TelnyxFrameSerializer. Parameters: @@ -82,10 +82,26 @@ class TelnyxFrameSerializer(FrameSerializer): api_key: Your Telnyx API key (required for auto hang-up). params: Configuration parameters. """ + params = params or TelnyxFrameSerializer.InputParams() + super().__init__(params) + self._params: TelnyxFrameSerializer.InputParams = params + + # Validate hangup-related parameters if auto_hang_up is enabled + if self._params.auto_hang_up: + missing_credentials = [] + if not call_control_id: + missing_credentials.append("call_control_id") + if not api_key: + missing_credentials.append("api_key") + + if missing_credentials: + raise ValueError( + f"auto_hang_up is enabled but missing required parameters: {', '.join(missing_credentials)}" + ) + self._stream_id = stream_id self._call_control_id = call_control_id self._api_key = api_key - self._params = params or TelnyxFrameSerializer.InputParams() self._params.outbound_encoding = outbound_encoding self._params.inbound_encoding = inbound_encoding @@ -163,14 +179,10 @@ class TelnyxFrameSerializer(FrameSerializer): async def _hang_up_call(self): """Hang up the Telnyx call using Telnyx's REST API.""" try: - call_control_id = self._call_control_id - api_key = self._api_key - - if not call_control_id or not api_key: - logger.warning( - "Cannot hang up Telnyx call: call_control_id and api_key must be provided" - ) - return + # __init__ guarantees these are non-None whenever auto_hang_up is True, + # which is the only path that reaches this method. + call_control_id = cast(str, self._call_control_id) + api_key = cast(str, self._api_key) # Telnyx API endpoint for hanging up a call endpoint = f"https://api.telnyx.com/v2/calls/{call_control_id}/actions/hangup" diff --git a/src/pipecat/serializers/twilio.py b/src/pipecat/serializers/twilio.py index 857610b4e..d93543c8e 100644 --- a/src/pipecat/serializers/twilio.py +++ b/src/pipecat/serializers/twilio.py @@ -8,6 +8,7 @@ import base64 import json +from typing import cast from loguru import logger @@ -75,7 +76,9 @@ class TwilioFrameSerializer(FrameSerializer): edge: Twilio edge location (e.g., "sydney", "dublin"). Must be specified with region. params: Configuration parameters. """ - super().__init__(params or TwilioFrameSerializer.InputParams()) + params = params or TwilioFrameSerializer.InputParams() + super().__init__(params) + self._params: TwilioFrameSerializer.InputParams = params # Validate hangup-related parameters if auto_hang_up is enabled if self._params.auto_hang_up: @@ -178,9 +181,11 @@ class TwilioFrameSerializer(FrameSerializer): try: import aiohttp - account_sid = self._account_sid - auth_token = self._auth_token - call_sid = self._call_sid + # __init__ guarantees these are non-None whenever auto_hang_up is True, + # which is the only path that reaches this method. + account_sid = cast(str, self._account_sid) + auth_token = cast(str, self._auth_token) + call_sid = cast(str, self._call_sid) region = self._region edge = self._edge diff --git a/src/pipecat/serializers/vonage.py b/src/pipecat/serializers/vonage.py index d778cf62c..7d82a717c 100644 --- a/src/pipecat/serializers/vonage.py +++ b/src/pipecat/serializers/vonage.py @@ -54,7 +54,9 @@ class VonageFrameSerializer(FrameSerializer): Args: params: Configuration parameters. """ - super().__init__(params or VonageFrameSerializer.InputParams()) + params = params or VonageFrameSerializer.InputParams() + super().__init__(params) + self._params: VonageFrameSerializer.InputParams = params self._vonage_sample_rate = self._params.vonage_sample_rate self._sample_rate = 0 # Pipeline input rate From 263cad41f096c986a91a2c357985d7b1796ea0c3 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 21 Apr 2026 18:14:15 -0400 Subject: [PATCH 47/49] Add changelog for #4349 --- changelog/4349.changed.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/4349.changed.md diff --git a/changelog/4349.changed.md b/changelog/4349.changed.md new file mode 100644 index 000000000..d41e1edb1 --- /dev/null +++ b/changelog/4349.changed.md @@ -0,0 +1 @@ +- ⚠️ `PlivoFrameSerializer` and `TelnyxFrameSerializer` now raise `ValueError` at construction when `auto_hang_up=True` (the default) but required credentials are missing, matching `TwilioFrameSerializer`. Previously they constructed successfully and the hangup failed silently at call-end, leaving phantom billable sessions on the provider. If you relied on the old silent behavior, pass `auto_hang_up=False` explicitly or provide the credentials. The specific fields checked are `call_id`/`auth_id`/`auth_token` for Plivo and `call_control_id`/`api_key` for Telnyx. From d953c201bd857ed030ed508c27af676406557313 Mon Sep 17 00:00:00 2001 From: filipi87 Date: Wed, 22 Apr 2026 10:24:21 -0300 Subject: [PATCH 48/49] Adding changelog entry to the fix. --- changelog/4323.fixed.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/4323.fixed.md diff --git a/changelog/4323.fixed.md b/changelog/4323.fixed.md new file mode 100644 index 000000000..26929a095 --- /dev/null +++ b/changelog/4323.fixed.md @@ -0,0 +1 @@ +- Fixed whitespace handling in TTS token streaming mode. Inter-token whitespace (e.g., spaces between words) is now preserved for correct prosody, while leading whitespace before the first non-whitespace token is still stripped to avoid issues with TTS models that are sensitive to leading spaces. From 7bba74ebd6b5196be5f9fe027f103ef9712f531b Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 21 Apr 2026 18:51:30 -0400 Subject: [PATCH 49/49] Expand pyright coverage to full src/pipecat with per-file ignores Previously, six modules (adapters, audio, processors, serializers, services, transports) were ignored wholesale. Many files in those modules already pass type checking, but we had no way to protect them from regressions or make the remaining work visible. Switch the include list to src/pipecat so any new module is checked by default, and replace directory-level ignores with the 140 specific files that still fail. This puts 189 previously-untyped files under type checking immediately and turns the remaining work into a concrete, shrinking TODO list. --- pyrightconfig.json | 157 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 136 insertions(+), 21 deletions(-) diff --git a/pyrightconfig.json b/pyrightconfig.json index cd1a69be2..74682c3dc 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -2,29 +2,144 @@ "typeCheckingMode": "basic", "pythonVersion": "3.11", "pythonPlatform": "All", - "include": [ - "scripts", - "src/pipecat/clocks", - "src/pipecat/extensions", - "src/pipecat/frames", - "src/pipecat/metrics", - "src/pipecat/observers", - "src/pipecat/pipeline", - "src/pipecat/runner", - "src/pipecat/serializers", - "src/pipecat/tests", - "src/pipecat/transcriptions", - "src/pipecat/turns", - "src/pipecat/utils" - ], + "include": ["scripts", "src/pipecat"], "exclude": ["**/*_pb2.py", "**/__pycache__"], "ignore": [ - "src/pipecat/adapters", - "src/pipecat/audio", - "src/pipecat/processors", - "src/pipecat/services", - "src/pipecat/transports", - "tests" + "tests", + "src/pipecat/adapters/base_llm_adapter.py", + "src/pipecat/adapters/schemas/direct_function.py", + "src/pipecat/adapters/schemas/tools_schema.py", + "src/pipecat/adapters/services/anthropic_adapter.py", + "src/pipecat/adapters/services/aws_nova_sonic_adapter.py", + "src/pipecat/adapters/services/bedrock_adapter.py", + "src/pipecat/adapters/services/gemini_adapter.py", + "src/pipecat/adapters/services/grok_realtime_adapter.py", + "src/pipecat/adapters/services/inworld_realtime_adapter.py", + "src/pipecat/adapters/services/open_ai_adapter.py", + "src/pipecat/adapters/services/open_ai_realtime_adapter.py", + "src/pipecat/adapters/services/open_ai_responses_adapter.py", + "src/pipecat/adapters/services/perplexity_adapter.py", + "src/pipecat/audio/dtmf/utils.py", + "src/pipecat/audio/filters/aic_filter.py", + "src/pipecat/audio/filters/krisp_viva_filter.py", + "src/pipecat/audio/filters/rnnoise_filter.py", + "src/pipecat/audio/resamplers/soxr_stream_resampler.py", + "src/pipecat/audio/turn/smart_turn/local_smart_turn_v2.py", + "src/pipecat/audio/turn/smart_turn/local_smart_turn_v3.py", + "src/pipecat/audio/vad/silero.py", + "src/pipecat/processors/aggregators/llm_context.py", + "src/pipecat/processors/aggregators/llm_response_universal.py", + "src/pipecat/processors/frame_processor.py", + "src/pipecat/processors/frameworks/langchain.py", + "src/pipecat/processors/frameworks/rtvi/observer.py", + "src/pipecat/processors/frameworks/rtvi/processor.py", + "src/pipecat/processors/frameworks/strands_agents.py", + "src/pipecat/processors/gstreamer/pipeline_source.py", + "src/pipecat/processors/metrics/frame_processor_metrics.py", + "src/pipecat/services/ai_service.py", + "src/pipecat/services/anthropic/llm.py", + "src/pipecat/services/assemblyai/stt.py", + "src/pipecat/services/asyncai/tts.py", + "src/pipecat/services/aws/agent_core.py", + "src/pipecat/services/aws/llm.py", + "src/pipecat/services/aws/nova_sonic/llm.py", + "src/pipecat/services/aws/sagemaker/bidi_client.py", + "src/pipecat/services/aws/stt.py", + "src/pipecat/services/aws/tts.py", + "src/pipecat/services/aws/utils.py", + "src/pipecat/services/azure/stt.py", + "src/pipecat/services/azure/tts.py", + "src/pipecat/services/camb/tts.py", + "src/pipecat/services/cartesia/stt.py", + "src/pipecat/services/cartesia/tts.py", + "src/pipecat/services/deepgram/flux/base.py", + "src/pipecat/services/deepgram/flux/sagemaker/stt.py", + "src/pipecat/services/deepgram/flux/stt.py", + "src/pipecat/services/deepgram/sagemaker/stt.py", + "src/pipecat/services/deepgram/sagemaker/tts.py", + "src/pipecat/services/deepgram/stt.py", + "src/pipecat/services/deepgram/tts.py", + "src/pipecat/services/elevenlabs/stt.py", + "src/pipecat/services/elevenlabs/tts.py", + "src/pipecat/services/fish/tts.py", + "src/pipecat/services/gladia/stt.py", + "src/pipecat/services/google/gemini_live/llm.py", + "src/pipecat/services/google/gemini_live/vertex/llm.py", + "src/pipecat/services/google/image.py", + "src/pipecat/services/google/llm.py", + "src/pipecat/services/google/stt.py", + "src/pipecat/services/google/tts.py", + "src/pipecat/services/google/vertex/llm.py", + "src/pipecat/services/gradium/stt.py", + "src/pipecat/services/gradium/tts.py", + "src/pipecat/services/groq/tts.py", + "src/pipecat/services/heygen/api_interactive_avatar.py", + "src/pipecat/services/heygen/base_api.py", + "src/pipecat/services/heygen/client.py", + "src/pipecat/services/heygen/video.py", + "src/pipecat/services/hume/tts.py", + "src/pipecat/services/image_service.py", + "src/pipecat/services/inworld/realtime/llm.py", + "src/pipecat/services/inworld/tts.py", + "src/pipecat/services/kokoro/tts.py", + "src/pipecat/services/llm_service.py", + "src/pipecat/services/lmnt/tts.py", + "src/pipecat/services/mcp_service.py", + "src/pipecat/services/mem0/memory.py", + "src/pipecat/services/mistral/llm.py", + "src/pipecat/services/mistral/stt.py", + "src/pipecat/services/mistral/tts.py", + "src/pipecat/services/moondream/vision.py", + "src/pipecat/services/neuphonic/tts.py", + "src/pipecat/services/nvidia/stt.py", + "src/pipecat/services/nvidia/tts.py", + "src/pipecat/services/openai/base_llm.py", + "src/pipecat/services/openai/image.py", + "src/pipecat/services/openai/llm.py", + "src/pipecat/services/openai/realtime/llm.py", + "src/pipecat/services/openai/responses/llm.py", + "src/pipecat/services/openai/stt.py", + "src/pipecat/services/openai/tts.py", + "src/pipecat/services/openrouter/llm.py", + "src/pipecat/services/piper/tts.py", + "src/pipecat/services/resembleai/tts.py", + "src/pipecat/services/rime/tts.py", + "src/pipecat/services/sambanova/llm.py", + "src/pipecat/services/sarvam/llm.py", + "src/pipecat/services/sarvam/stt.py", + "src/pipecat/services/sarvam/tts.py", + "src/pipecat/services/simli/video.py", + "src/pipecat/services/smallest/stt.py", + "src/pipecat/services/smallest/tts.py", + "src/pipecat/services/soniox/stt.py", + "src/pipecat/services/speechmatics/stt.py", + "src/pipecat/services/speechmatics/tts.py", + "src/pipecat/services/stt_service.py", + "src/pipecat/services/tavus/video.py", + "src/pipecat/services/tts_service.py", + "src/pipecat/services/ultravox/llm.py", + "src/pipecat/services/vision_service.py", + "src/pipecat/services/websocket_service.py", + "src/pipecat/services/whisper/stt.py", + "src/pipecat/services/xai/realtime/events.py", + "src/pipecat/services/xai/realtime/llm.py", + "src/pipecat/services/xai/stt.py", + "src/pipecat/services/xai/tts.py", + "src/pipecat/services/xtts/tts.py", + "src/pipecat/transports/base_output.py", + "src/pipecat/transports/daily/transport.py", + "src/pipecat/transports/heygen/transport.py", + "src/pipecat/transports/lemonslice/transport.py", + "src/pipecat/transports/livekit/transport.py", + "src/pipecat/transports/local/tk.py", + "src/pipecat/transports/smallwebrtc/connection.py", + "src/pipecat/transports/smallwebrtc/request_handler.py", + "src/pipecat/transports/smallwebrtc/transport.py", + "src/pipecat/transports/tavus/transport.py", + "src/pipecat/transports/websocket/client.py", + "src/pipecat/transports/websocket/fastapi.py", + "src/pipecat/transports/websocket/server.py", + "src/pipecat/transports/whatsapp/client.py" ], "reportMissingImports": false }