diff --git a/.github/workflows/format.yaml b/.github/workflows/format.yaml index 8284be043..7ac009bdc 100644 --- a/.github/workflows/format.yaml +++ b/.github/workflows/format.yaml @@ -32,7 +32,7 @@ jobs: run: uv python install 3.12 - name: Install development dependencies - run: uv sync --group dev + run: uv sync --group dev --extra daily --extra tracing - name: Ruff formatter id: ruff-format @@ -41,3 +41,7 @@ jobs: - name: Ruff linter (all rules) id: ruff-check run: uv run ruff check + + - name: Type check (pyright) + id: pyright + run: uv run pyright diff --git a/.github/workflows/update-docs.yml b/.github/workflows/update-docs.yml index d26862766..bb0227233 100644 --- a/.github/workflows/update-docs.yml +++ b/.github/workflows/update-docs.yml @@ -114,6 +114,7 @@ jobs: GH_TOKEN=$DOCS_SYNC_TOKEN gh pr create \ --repo pipecat-ai/docs \ --label auto-docs \ + --label pipecat \ --title "docs: update for pipecat PR #${{ steps.pr.outputs.number }}" \ --body "$(cat <<'BODY' Automated documentation update for [pipecat PR #${{ steps.pr.outputs.number }}](https://github.com/pipecat-ai/pipecat/pull/${{ steps.pr.outputs.number }}). diff --git a/README.md b/README.md index 15be8d3eb..5ea9fc4b9 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ Catch new features, interviews, and how-tos on our [Pipecat TV](https://www.yout | Category | Services | | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Gradium](https://docs.pipecat.ai/server/services/stt/gradium), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [Mistral](https://docs.pipecat.ai/server/services/stt/mistral), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Gradium](https://docs.pipecat.ai/server/services/stt/gradium), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [Mistral](https://docs.pipecat.ai/server/services/stt/mistral), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper), [xAI](https://docs.pipecat.ai/server/services/stt/xai) | | LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [Nebius](https://docs.pipecat.ai/server/services/llm/nebius), [Novita](https://docs.pipecat.ai/server/services/llm/novita), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nvidia), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/llm/sarvam), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | | Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Camb AI](https://docs.pipecat.ai/server/services/tts/camb), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Gradium](https://docs.pipecat.ai/server/services/tts/gradium), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [Kokoro](https://docs.pipecat.ai/server/services/tts/kokoro), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Mistral](https://docs.pipecat.ai/server/services/tts/mistral), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [Resemble](https://docs.pipecat.ai/server/services/tts/resemble), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [Smallest](https://docs.pipecat.ai/server/services/tts/smallest), [Speechmatics](https://docs.pipecat.ai/server/services/tts/speechmatics), [xAI](https://docs.pipecat.ai/server/services/tts/xai), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | | Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [Grok Voice Agent](https://docs.pipecat.ai/server/services/s2s/grok), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai), [Ultravox](https://docs.pipecat.ai/server/services/s2s/ultravox), | diff --git a/changelog/4270.changed.md b/changelog/4270.changed.md new file mode 100644 index 000000000..eb9f54972 --- /dev/null +++ b/changelog/4270.changed.md @@ -0,0 +1 @@ +- Updated `NvidiaLLMService` to emit model reasoning as `LLMThought*Frame`s (from both `reasoning_content` and `...` output), avoid mixing reasoning text into normal assistant content, and allow keyless local NIM endpoints while warning when the cloud endpoint is used without an API key. diff --git a/changelog/4320.fixed.md b/changelog/4320.fixed.md new file mode 100644 index 000000000..12e49c957 --- /dev/null +++ b/changelog/4320.fixed.md @@ -0,0 +1 @@ +- Fixed `SmallestTTSService` WebSocket endpoint URL to match Smallest AI v4.0.0 API (`wss://waves-api.smallest.ai` → `wss://api.smallest.ai`) and restored keepalive using a silent space message instead of the unsupported flush command. diff --git a/changelog/4323.fixed.md b/changelog/4323.fixed.md new file mode 100644 index 000000000..26929a095 --- /dev/null +++ b/changelog/4323.fixed.md @@ -0,0 +1 @@ +- Fixed whitespace handling in TTS token streaming mode. Inter-token whitespace (e.g., spaces between words) is now preserved for correct prosody, while leading whitespace before the first non-whitespace token is still stripped to avoid issues with TTS models that are sensitive to leading spaces. diff --git a/changelog/4324.added.md b/changelog/4324.added.md new file mode 100644 index 000000000..d38887585 --- /dev/null +++ b/changelog/4324.added.md @@ -0,0 +1 @@ +- Added incremental `pyright` type checking. A `pyrightconfig.json` at the repo root uses `typeCheckingMode: "basic"` with an explicit `include` list of modules that pass cleanly (`clocks`, `metrics`, `transcriptions`, `frames`, `observers`, `extensions`, `turns`, `pipeline`, `runner`). Remaining modules will be added in subsequent PRs. CI enforces the checked set via `uv run pyright` in the format workflow. diff --git a/changelog/4324.changed.md b/changelog/4324.changed.md new file mode 100644 index 000000000..672c60b62 --- /dev/null +++ b/changelog/4324.changed.md @@ -0,0 +1 @@ +- `LiveKitRunnerArguments.token` is now a required `str` (previously `str | None` with a default of `None`). LiveKit requires a token to join a room, so the type now reflects reality. This only affects custom runners that construct `LiveKitRunnerArguments` directly; code consuming the argument from the standard runner is unaffected. diff --git a/changelog/4325.fixed.md b/changelog/4325.fixed.md new file mode 100644 index 000000000..b87a97bfb --- /dev/null +++ b/changelog/4325.fixed.md @@ -0,0 +1 @@ +- Fixed `SentryMetrics` silently dropping `MetricsFrame`s from `stop_ttfb_metrics` and `stop_processing_metrics`. `SentryMetrics` called the base `FrameProcessorMetrics` implementation but discarded its return value, so `FrameProcessor` never pushed the `MetricsFrame` downstream. This prevented observers (e.g. `UserBotLatencyObserver`, `MetricsLogObserver`) from seeing TTFB and processing metrics for any service using `metrics=SentryMetrics()`. The metrics were still calculated and Sentry transactions still completed — only the downstream frame push was affected. diff --git a/changelog/4326.added.md b/changelog/4326.added.md new file mode 100644 index 000000000..856c7b5d7 --- /dev/null +++ b/changelog/4326.added.md @@ -0,0 +1 @@ +- Added multilingual support to `DeepgramFluxSTTService` via a new `language_hints: list[Language]` setting. Works with Deepgram's new `flux-general-multi` model to bias transcription across English, Spanish, French, German, Hindi, Russian, Portuguese, Japanese, Italian, and Dutch. Omit the hints to use auto-detection, or pass a subset to bias toward expected languages. Hints can be updated mid-stream via `STTUpdateSettingsFrame` (sent as a Deepgram `Configure` control message, no reconnect) to support detect-then-lock flows. diff --git a/changelog/4326.changed.md b/changelog/4326.changed.md new file mode 100644 index 000000000..4cc6e9bb7 --- /dev/null +++ b/changelog/4326.changed.md @@ -0,0 +1 @@ +- `TranscriptionFrame.language` and `InterimTranscriptionFrame.language` emitted by `DeepgramFluxSTTService` now reflect the language Deepgram detected for each turn (read from the `languages` field on Flux's `TurnInfo` event). On `flux-general-multi` this gives per-turn accuracy for downstream consumers (e.g. TTS voice selection). `flux-general-en` continues to emit `Language.EN`. diff --git a/changelog/4334.added.md b/changelog/4334.added.md new file mode 100644 index 000000000..e3dc45543 --- /dev/null +++ b/changelog/4334.added.md @@ -0,0 +1 @@ +- Added fine-grained server-side VAD tuning options to `SarvamSTTService.Settings` for the `saaras:v3` model, including speech thresholds, frame-count controls, pre-speech padding, interruption sensitivity, and initial-frame skipping. \ No newline at end of file diff --git a/changelog/4334.changed.md b/changelog/4334.changed.md new file mode 100644 index 000000000..ff8d75147 --- /dev/null +++ b/changelog/4334.changed.md @@ -0,0 +1 @@ +- `SarvamSTTService` now uses `saaras:v3` as its default model instead of `saarika:v2.5`. Applications that relied on the previous default should set `settings=SarvamSTTService.Settings(model="saarika:v2.5")` explicitly. \ No newline at end of file diff --git a/changelog/4337.changed.2.md b/changelog/4337.changed.2.md new file mode 100644 index 000000000..a1da061c8 --- /dev/null +++ b/changelog/4337.changed.2.md @@ -0,0 +1 @@ +- `SpeechTimeoutUserTurnStopStrategy` now waits only `user_speech_timeout` when a transcript arrives without a VAD stop event, rather than `max(ttfs_p99_latency, user_speech_timeout)`. If you had `ttfs_p99_latency > user_speech_timeout`, turn detection in that path is slightly faster than before. diff --git a/changelog/4337.changed.md b/changelog/4337.changed.md new file mode 100644 index 000000000..67c1ddbfe --- /dev/null +++ b/changelog/4337.changed.md @@ -0,0 +1 @@ +- If you use an STT service that emits finalized transcripts (Speechmatics, Soniox, Deepgram Flux, AssemblyAI) with `SpeechTimeoutUserTurnStopStrategy`, user turns now end as soon as `user_speech_timeout` elapses after VAD stop. Previously the strategy also waited for the STT P99 latency (`ttfs_p99_latency`) even when the transcript was already marked final. `user_speech_timeout` is still honored as a floor — STT finalization never shortens it. diff --git a/changelog/4340.added.md b/changelog/4340.added.md new file mode 100644 index 000000000..5abcc0f0e --- /dev/null +++ b/changelog/4340.added.md @@ -0,0 +1 @@ +- Added `XAISTTService` for real-time speech-to-text using xAI's voice STT WebSocket API (`wss://api.x.ai/v1/stt`). Streams raw audio (PCM, µ-law, or A-law) and emits interim and final transcription frames driven by the server's `is_final` / `speech_final` flags. Settings expose `interim_results`, `endpointing`, `language`, `multichannel`, `channels`, and `diarize`. Requires the `xai` optional extra (`pip install "pipecat-ai[xai]"`). diff --git a/changelog/4341.added.md b/changelog/4341.added.md new file mode 100644 index 000000000..9301bcc5f --- /dev/null +++ b/changelog/4341.added.md @@ -0,0 +1 @@ +- Added `XAITTSService` for streaming text-to-speech using xAI's WebSocket TTS endpoint (`wss://api.x.ai/v1/tts`). Streams `text.delta` chunks up and base64 `audio.delta` chunks down on the same connection so audio begins flowing before the full utterance finishes synthesizing; complements the batch-HTTP `XAIHttpTTSService`. Defaults to raw PCM output so `TTSAudioRawFrame` needs no decoding. The `xai` optional extra now pulls in `pipecat-ai[websockets-base]`. diff --git a/changelog/4344.fixed.md b/changelog/4344.fixed.md new file mode 100644 index 000000000..4e786f65c --- /dev/null +++ b/changelog/4344.fixed.md @@ -0,0 +1 @@ +- Fixed `ElevenLabsTTSService` and `ElevenLabsHttpTTSService` emitting word timestamps and `TTSTextFrame` content that matched the input text instead of the spoken audio when a pronunciation dictionary (`pronunciation_dictionary_locators`) or text normalization rewrote the input. Both services now consume ElevenLabs' normalized alignment, so downstream consumers (captions, transcripts, context aggregation) reflect what the listener actually hears. diff --git a/changelog/4347.fixed.md b/changelog/4347.fixed.md new file mode 100644 index 000000000..e5c8206e5 --- /dev/null +++ b/changelog/4347.fixed.md @@ -0,0 +1 @@ +- Fixed a crash in `DeepgramSTTService` when an `STTUpdateSettingsFrame` arrived before the WebSocket handshake completed (for example, when pushing an update upstream on `StartFrame`). The settings-triggered reconnect cancelled the in-flight connection task before its keepalive task was created, causing an `UnboundLocalError: cannot access local variable 'keepalive_task'` in the handler's `finally` block. diff --git a/changelog/4349.changed.md b/changelog/4349.changed.md new file mode 100644 index 000000000..d41e1edb1 --- /dev/null +++ b/changelog/4349.changed.md @@ -0,0 +1 @@ +- ⚠️ `PlivoFrameSerializer` and `TelnyxFrameSerializer` now raise `ValueError` at construction when `auto_hang_up=True` (the default) but required credentials are missing, matching `TwilioFrameSerializer`. Previously they constructed successfully and the hangup failed silently at call-end, leaving phantom billable sessions on the provider. If you relied on the old silent behavior, pass `auto_hang_up=False` explicitly or provide the credentials. The specific fields checked are `call_id`/`auth_id`/`auth_token` for Plivo and `call_control_id`/`api_key` for Telnyx. diff --git a/examples/audio/audio-bot-background-sound.py b/examples/audio/audio-bot-background-sound.py index 4237bb625..6417e5549 100644 --- a/examples/audio/audio-bot-background-sound.py +++ b/examples/audio/audio-bot-background-sound.py @@ -71,17 +71,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/audio/audio-recording.py b/examples/audio/audio-recording.py index 68354e0dd..9ece2b796 100644 --- a/examples/audio/audio-recording.py +++ b/examples/audio/audio-recording.py @@ -108,17 +108,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"), audio_passthrough=True) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"], audio_passthrough=True) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/audio/audio-sound-effects.py b/examples/audio/audio-sound-effects.py index 1f7fdd339..19ed85675 100644 --- a/examples/audio/audio-sound-effects.py +++ b/examples/audio/audio-sound-effects.py @@ -102,17 +102,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), diff --git a/examples/context-summarization/context-summarization-dedicated-llm.py b/examples/context-summarization/context-summarization-dedicated-llm.py index e4de35cb7..bc13e9c46 100644 --- a/examples/context-summarization/context-summarization-dedicated-llm.py +++ b/examples/context-summarization/context-summarization-dedicated-llm.py @@ -89,10 +89,10 @@ async def get_current_weather(params: FunctionCallParams): async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info("Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -109,7 +109,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Primary LLM for conversation (could be any provider) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction=system_prompt, ), @@ -117,7 +117,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Dedicated cheap/fast LLM for summarization only summarization_llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( model="gemini-2.5-flash", ), diff --git a/examples/context-summarization/context-summarization-google.py b/examples/context-summarization/context-summarization-google.py index f8cf50419..3b0daca32 100644 --- a/examples/context-summarization/context-summarization-google.py +++ b/examples/context-summarization/context-summarization-google.py @@ -77,17 +77,17 @@ async def get_current_weather(params: FunctionCallParams): async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info("Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You have access to tools to get the current weather - use them when relevant.", ), diff --git a/examples/context-summarization/context-summarization-manual-openai.py b/examples/context-summarization/context-summarization-manual-openai.py index 017831695..23f70d97c 100644 --- a/examples/context-summarization/context-summarization-manual-openai.py +++ b/examples/context-summarization/context-summarization-manual-openai.py @@ -72,10 +72,10 @@ async def summarize_conversation(params: FunctionCallParams): async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info("Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -91,7 +91,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): """ llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction=system_prompt, ), diff --git a/examples/context-summarization/context-summarization-openai.py b/examples/context-summarization/context-summarization-openai.py index 53c5b26a5..05566f32b 100644 --- a/examples/context-summarization/context-summarization-openai.py +++ b/examples/context-summarization/context-summarization-openai.py @@ -77,17 +77,17 @@ async def get_current_weather(params: FunctionCallParams): async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info("Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You have access to tools to get the current weather - use them when relevant.", ), diff --git a/examples/features/features-before-and-after-events.py b/examples/features/features-before-and-after-events.py index b9111b945..dedf3c4fa 100644 --- a/examples/features/features-before-and-after-events.py +++ b/examples/features/features-before-and-after-events.py @@ -63,17 +63,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/features/features-concurrent-llm-evaluation.py b/examples/features/features-concurrent-llm-evaluation.py index 25b9372d0..cb085a8c7 100644 --- a/examples/features/features-concurrent-llm-evaluation.py +++ b/examples/features/features-concurrent-llm-evaluation.py @@ -58,24 +58,24 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) openai_llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), ) groq_llm = GroqLLMService( - api_key=os.getenv("GROQ_API_KEY"), + api_key=os.environ["GROQ_API_KEY"], settings=GroqLLMService.Settings( system_instruction="You are a very helpful assistant. Your goal is to demonstrate your capabilities in detail in a creative and helpful way.", ), diff --git a/examples/features/features-concurrent-llm-rtvi-ignored-sources.py b/examples/features/features-concurrent-llm-rtvi-ignored-sources.py index 678c56a0b..70530b51e 100644 --- a/examples/features/features-concurrent-llm-rtvi-ignored-sources.py +++ b/examples/features/features-concurrent-llm-rtvi-ignored-sources.py @@ -63,10 +63,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info("Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -74,7 +74,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Main LLM — drives the conversation. Its RTVI events reach the client. main_llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), @@ -83,7 +83,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Evaluator LLM — silently grades the user's message in the background. # Its RTVI events will be suppressed so the client is unaware of this branch. evaluator_llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], name="EvaluatorLLM", settings=OpenAILLMService.Settings( system_instruction="You are a silent quality evaluator. When given a user message, respond with a single JSON object: {'score': <1-5>, 'reason': ''}. Do not respond conversationally.", diff --git a/examples/features/features-custom-frame-processor.py b/examples/features/features-custom-frame-processor.py index bf143c40d..ab82bae80 100644 --- a/examples/features/features-custom-frame-processor.py +++ b/examples/features/features-custom-frame-processor.py @@ -91,17 +91,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/features/features-gpu-container-local-bot.py b/examples/features/features-gpu-container-local-bot.py index c61870866..67a8dabc7 100644 --- a/examples/features/features-gpu-container-local-bot.py +++ b/examples/features/features-gpu-container-local-bot.py @@ -56,10 +56,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = DeepgramTTSService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], settings=DeepgramTTSService.Settings( voice="aura-asteria-en", ), @@ -68,7 +68,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = OpenAILLMService( # To use OpenAI - # api_key=os.getenv("OPENAI_API_KEY"), + # api_key=os.environ["OPENAI_API_KEY"], # Or, to use a local vLLM (or similar) api server settings=OpenAILLMService.Settings( model="meta-llama/Meta-Llama-3-8B-Instruct", diff --git a/examples/features/features-live-translation.py b/examples/features/features-live-translation.py index 7a4a4b112..e4e7d075f 100644 --- a/examples/features/features-live-translation.py +++ b/examples/features/features-live-translation.py @@ -55,17 +55,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="d4db5fb9-f44b-4bd1-85fa-192e0f0d75f9", # Spanish-speaking Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a live translation assistant. Your sole purpose is to translate English text into Spanish. When you receive English text from the user, immediately translate it into natural, fluent Spanish. Do not add explanations, commentary, or extra information—only provide the Spanish translation of the text you receive.", ), diff --git a/examples/features/features-pattern-pair-voice-switching.py b/examples/features/features-pattern-pair-voice-switching.py index 38926d915..1f5b83cb8 100644 --- a/examples/features/features-pattern-pair-voice-switching.py +++ b/examples/features/features-pattern-pair-voice-switching.py @@ -126,14 +126,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm_text_aggregator.on_pattern_match("voice", on_voice_tag) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) # Process LLM text through the pattern aggregator before TTS llm_text_processor = LLMTextProcessor(text_aggregator=llm_text_aggregator) # Initialize TTS with narrator voice as default tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice=VOICE_IDS["narrator"], ), @@ -190,7 +190,7 @@ Remember: Use narrator voice for EVERYTHING except the actual quoted dialogue."" # Initialize LLM llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction=system_prompt, ), diff --git a/examples/features/features-service-switcher.py b/examples/features/features-service-switcher.py index 9ded72a60..c2c78738a 100644 --- a/examples/features/features-service-switcher.py +++ b/examples/features/features-service-switcher.py @@ -94,19 +94,19 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): required=["location", "format"], ) - stt_cartesia = CartesiaSTTService(api_key=os.getenv("CARTESIA_API_KEY")) - stt_deepgram = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt_cartesia = CartesiaSTTService(api_key=os.environ["CARTESIA_API_KEY"]) + stt_deepgram = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) # Uses ServiceSwitcherStrategyManual by default stt_switcher = ServiceSwitcher(services=[stt_cartesia, stt_deepgram]) tts_cartesia = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) tts_deepgram = DeepgramTTSService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], settings=DeepgramTTSService.Settings( voice="aura-2-helena-en", ), @@ -117,11 +117,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): system_prompt = "You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way." llm_openai = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings(system_instruction=system_prompt), ) llm_google = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings(system_instruction=system_prompt), ) # Uses ServiceSwitcherStrategyManual by default diff --git a/examples/features/features-switch-languages.py b/examples/features/features-switch-languages.py index c47d66847..f99de0aec 100644 --- a/examples/features/features-switch-languages.py +++ b/examples/features/features-switch-languages.py @@ -42,14 +42,14 @@ class SwitchLanguage(ParallelPipeline): self._current_language = "English" english_tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) spanish_tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="d4db5fb9-f44b-4bd1-85fa-192e0f0d75f9", # Spanish-speaking Lady ), @@ -101,7 +101,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = DeepgramSTTService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], settings=DeepgramSTTService.Settings( language="multi", ), @@ -110,7 +110,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): tts = SwitchLanguage() llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You can speak the following languages: 'English' and 'Spanish'.", ), diff --git a/examples/features/features-switch-voices.py b/examples/features/features-switch-voices.py index 0c47d69ae..d33a10e8f 100644 --- a/examples/features/features-switch-voices.py +++ b/examples/features/features-switch-voices.py @@ -42,21 +42,21 @@ class SwitchVoices(ParallelPipeline): self._current_voice = "News Lady" news_lady = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="bf991597-6c13-47e4-8411-91ec2de5c466", # Newslady ), ) british_lady = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) barbershop_man = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="a0e99841-438c-4a64-b679-ae501e7d6091", # Barbershop Man ), @@ -114,12 +114,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = SwitchVoices() llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative and helpful way. You can do the following voices: 'News Lady', 'British Lady' and 'Barbershop Man'.", ), diff --git a/examples/features/features-user-email-gathering.py b/examples/features/features-user-email-gathering.py index 7e8a69011..f0ec3160f 100644 --- a/examples/features/features-user-email-gathering.py +++ b/examples/features/features-user-email-gathering.py @@ -60,13 +60,13 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) # Cartesia offers a `` tags that we can use to ask the user # to confirm the emails. # (see https://docs.cartesia.ai/build-with-sonic/formatting-text-for-sonic/spelling-out-input-text) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -84,7 +84,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You need to gather a valid email or emails from the user. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. If the user provides one or more email addresses confirm them with the user. Enclose all emails with tags, for example a@a.com.", ), diff --git a/examples/features/features-voicemail-detection.py b/examples/features/features-voicemail-detection.py index 854e546ec..4c8f07d6e 100644 --- a/examples/features/features-voicemail-detection.py +++ b/examples/features/features-voicemail-detection.py @@ -52,22 +52,22 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), ) - classifier_llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + classifier_llm = OpenAILLMService(api_key=os.environ["OPENAI_API_KEY"]) voicemail = VoicemailDetector(llm=classifier_llm) diff --git a/examples/features/features-wake-phrase.py b/examples/features/features-wake-phrase.py index 50f8b1fb8..8d39d5c60 100644 --- a/examples/features/features-wake-phrase.py +++ b/examples/features/features-wake-phrase.py @@ -57,21 +57,21 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = DeepgramSTTService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], settings=DeepgramSTTService.Settings( keyterm=["pipecat"], ), ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/function-calling/function-calling-anthropic-async-stream.py b/examples/function-calling/function-calling-anthropic-async-stream.py index 52069af68..b44bb5997 100644 --- a/examples/function-calling/function-calling-anthropic-async-stream.py +++ b/examples/function-calling/function-calling-anthropic-async-stream.py @@ -107,17 +107,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), + api_key=os.environ["ANTHROPIC_API_KEY"], enable_async_tool_cancellation=True, settings=AnthropicLLMService.Settings( system_instruction=( diff --git a/examples/function-calling/function-calling-anthropic-async.py b/examples/function-calling/function-calling-anthropic-async.py index 2c5fb9402..5046ec839 100644 --- a/examples/function-calling/function-calling-anthropic-async.py +++ b/examples/function-calling/function-calling-anthropic-async.py @@ -66,17 +66,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), + api_key=os.environ["ANTHROPIC_API_KEY"], enable_async_tool_cancellation=True, settings=AnthropicLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/function-calling/function-calling-anthropic-video.py b/examples/function-calling/function-calling-anthropic-video.py index c3a292f31..1b7617950 100644 --- a/examples/function-calling/function-calling-anthropic-video.py +++ b/examples/function-calling/function-calling-anthropic-video.py @@ -86,10 +86,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -97,7 +97,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Anthropic for vision analysis llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), + api_key=os.environ["ANTHROPIC_API_KEY"], settings=AnthropicLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You are able to describe images from the user camera.", ), diff --git a/examples/function-calling/function-calling-anthropic.py b/examples/function-calling/function-calling-anthropic.py index b8bb4eac6..3c4cf769a 100644 --- a/examples/function-calling/function-calling-anthropic.py +++ b/examples/function-calling/function-calling-anthropic.py @@ -65,17 +65,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), + api_key=os.environ["ANTHROPIC_API_KEY"], settings=AnthropicLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/function-calling/function-calling-aws-video.py b/examples/function-calling/function-calling-aws-video.py index 8db25671b..d6ae64438 100644 --- a/examples/function-calling/function-calling-aws-video.py +++ b/examples/function-calling/function-calling-aws-video.py @@ -86,10 +86,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), diff --git a/examples/function-calling/function-calling-azure.py b/examples/function-calling/function-calling-azure.py index 23c156da4..37d67b29b 100644 --- a/examples/function-calling/function-calling-azure.py +++ b/examples/function-calling/function-calling-azure.py @@ -60,18 +60,18 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = AzureLLMService( - api_key=os.getenv("AZURE_CHATGPT_API_KEY"), - endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + api_key=os.environ["AZURE_CHATGPT_API_KEY"], + endpoint=os.environ["AZURE_CHATGPT_ENDPOINT"], settings=AzureLLMService.Settings( model=os.getenv("AZURE_CHATGPT_MODEL"), system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/function-calling/function-calling-cerebras.py b/examples/function-calling/function-calling-cerebras.py index ad8475185..82d98d40e 100644 --- a/examples/function-calling/function-calling-cerebras.py +++ b/examples/function-calling/function-calling-cerebras.py @@ -60,17 +60,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = CerebrasLLMService( - api_key=os.getenv("CEREBRAS_API_KEY"), + api_key=os.environ["CEREBRAS_API_KEY"], settings=CerebrasLLMService.Settings( system_instruction="""You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. diff --git a/examples/function-calling/function-calling-deepseek.py b/examples/function-calling/function-calling-deepseek.py index f115299e2..fc354ccf5 100644 --- a/examples/function-calling/function-calling-deepseek.py +++ b/examples/function-calling/function-calling-deepseek.py @@ -60,17 +60,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = DeepSeekLLMService( - api_key=os.getenv("DEEPSEEK_API_KEY"), + api_key=os.environ["DEEPSEEK_API_KEY"], settings=DeepSeekLLMService.Settings( model="deepseek-chat", system_instruction="""You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. diff --git a/examples/function-calling/function-calling-direct.py b/examples/function-calling/function-calling-direct.py index 1c6ebe072..c885d6606 100644 --- a/examples/function-calling/function-calling-direct.py +++ b/examples/function-calling/function-calling-direct.py @@ -76,17 +76,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/function-calling/function-calling-fireworks.py b/examples/function-calling/function-calling-fireworks.py index 6ddb85981..8c681fa72 100644 --- a/examples/function-calling/function-calling-fireworks.py +++ b/examples/function-calling/function-calling-fireworks.py @@ -60,17 +60,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = FireworksLLMService( - api_key=os.getenv("FIREWORKS_API_KEY"), + api_key=os.environ["FIREWORKS_API_KEY"], settings=FireworksLLMService.Settings( model="accounts/fireworks/models/gpt-oss-20b", system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/function-calling/function-calling-google-async-stream.py b/examples/function-calling/function-calling-google-async-stream.py index 503880ca6..1660442b4 100644 --- a/examples/function-calling/function-calling-google-async-stream.py +++ b/examples/function-calling/function-calling-google-async-stream.py @@ -107,17 +107,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], enable_async_tool_cancellation=True, settings=GoogleLLMService.Settings( system_instruction=( diff --git a/examples/function-calling/function-calling-google-async.py b/examples/function-calling/function-calling-google-async.py index dbc86d664..f3c5fbc1d 100644 --- a/examples/function-calling/function-calling-google-async.py +++ b/examples/function-calling/function-calling-google-async.py @@ -98,10 +98,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -127,7 +127,7 @@ indicate you should use the get_image tool are: """ llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], enable_async_tool_cancellation=True, settings=GoogleLLMService.Settings( system_instruction=system_prompt, diff --git a/examples/function-calling/function-calling-google-vertex.py b/examples/function-calling/function-calling-google-vertex.py index 3bee2f1bb..290315f2c 100644 --- a/examples/function-calling/function-calling-google-vertex.py +++ b/examples/function-calling/function-calling-google-vertex.py @@ -60,19 +60,19 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = ElevenLabsTTSService( - api_key=os.getenv("ELEVENLABS_API_KEY", ""), + api_key=os.environ["ELEVENLABS_API_KEY"], settings=ElevenLabsTTSService.Settings( - voice=os.getenv("ELEVENLABS_VOICE_ID", ""), + voice=os.getenv("ELEVENLABS_VOICE_ID", "Xb7hH8MSUJpSbSDYk0k2"), ), ) llm = GoogleVertexLLMService( - credentials=os.getenv("GOOGLE_VERTEX_TEST_CREDENTIALS"), - project_id=os.getenv("GOOGLE_CLOUD_PROJECT_ID"), - location=os.getenv("GOOGLE_CLOUD_LOCATION"), + credentials=os.environ["GOOGLE_VERTEX_TEST_CREDENTIALS"], + project_id=os.environ["GOOGLE_CLOUD_PROJECT_ID"], + location=os.environ["GOOGLE_CLOUD_LOCATION"], settings=GoogleVertexLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), @@ -103,14 +103,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tools = ToolsSchema(standard_tools=[weather_function]) - messages = [ - { - "role": "developer", - "content": "Start a conversation with 'Hey there' to get the current weather.", - }, - ] - - context = LLMContext(messages, tools) + context = LLMContext(tools=tools) user_aggregator, assistant_aggregator = LLMContextAggregatorPair( context, user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), @@ -141,6 +134,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): async def on_client_connected(transport, client): logger.info(f"Client connected") # Kick off the conversation. + context.add_message( + { + "role": "developer", + "content": "Please introduce yourself to the user.", + } + ) await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") diff --git a/examples/function-calling/function-calling-google-video.py b/examples/function-calling/function-calling-google-video.py index 59816fdd5..65296c603 100644 --- a/examples/function-calling/function-calling-google-video.py +++ b/examples/function-calling/function-calling-google-video.py @@ -86,10 +86,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -97,7 +97,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Google Gemini model for vision analysis llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You are able to describe images from the user camera.", ), diff --git a/examples/function-calling/function-calling-google.py b/examples/function-calling/function-calling-google.py index 421a02a1e..1b6d5b152 100644 --- a/examples/function-calling/function-calling-google.py +++ b/examples/function-calling/function-calling-google.py @@ -96,10 +96,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -125,7 +125,7 @@ indicate you should use the get_image tool are: """ llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( system_instruction=system_prompt, ), diff --git a/examples/function-calling/function-calling-grok.py b/examples/function-calling/function-calling-grok.py index 05e1efaf9..537ab2025 100644 --- a/examples/function-calling/function-calling-grok.py +++ b/examples/function-calling/function-calling-grok.py @@ -62,10 +62,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = XAIHttpTTSService( - api_key=os.getenv("XAI_API_KEY"), + api_key=os.environ["XAI_API_KEY"], aiohttp_session=session, settings=XAIHttpTTSService.Settings( voice="eve", @@ -73,7 +73,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = GrokLLMService( - api_key=os.getenv("XAI_API_KEY"), + api_key=os.environ["XAI_API_KEY"], settings=GrokLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/function-calling/function-calling-groq.py b/examples/function-calling/function-calling-groq.py index 1e6e84338..d4c69349e 100644 --- a/examples/function-calling/function-calling-groq.py +++ b/examples/function-calling/function-calling-groq.py @@ -60,17 +60,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = GroqSTTService(api_key=os.getenv("GROQ_API_KEY")) + stt = GroqSTTService(api_key=os.environ["GROQ_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = GroqLLMService( - api_key=os.getenv("GROQ_API_KEY"), + api_key=os.environ["GROQ_API_KEY"], settings=GroqLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/function-calling/function-calling-mistral.py b/examples/function-calling/function-calling-mistral.py index c52d46c03..e44adf8ba 100644 --- a/examples/function-calling/function-calling-mistral.py +++ b/examples/function-calling/function-calling-mistral.py @@ -63,17 +63,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = MistralLLMService( - api_key=os.getenv("MISTRAL_API_KEY"), + api_key=os.environ["MISTRAL_API_KEY"], settings=MistralLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/function-calling/function-calling-moondream-video.py b/examples/function-calling/function-calling-moondream-video.py index b8076e7ce..56a0fa503 100644 --- a/examples/function-calling/function-calling-moondream-video.py +++ b/examples/function-calling/function-calling-moondream-video.py @@ -117,17 +117,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You are able to describe images from the user camera.", ), diff --git a/examples/function-calling/function-calling-nebius.py b/examples/function-calling/function-calling-nebius.py index 4fe9a378a..ed8fded2c 100644 --- a/examples/function-calling/function-calling-nebius.py +++ b/examples/function-calling/function-calling-nebius.py @@ -63,17 +63,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = NebiusLLMService( - api_key=os.getenv("NEBIUS_API_KEY"), + api_key=os.environ["NEBIUS_API_KEY"], settings=NebiusLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/function-calling/function-calling-novita.py b/examples/function-calling/function-calling-novita.py index da60d890b..d9838e37f 100644 --- a/examples/function-calling/function-calling-novita.py +++ b/examples/function-calling/function-calling-novita.py @@ -63,17 +63,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = NovitaLLMService( - api_key=os.getenv("NOVITA_API_KEY"), + api_key=os.environ["NOVITA_API_KEY"], settings=NovitaLLMService.Settings( model="openai/gpt-oss-120b", system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/function-calling/function-calling-nvidia.py b/examples/function-calling/function-calling-nvidia.py index e3db6db8d..2b4818ceb 100644 --- a/examples/function-calling/function-calling-nvidia.py +++ b/examples/function-calling/function-calling-nvidia.py @@ -60,17 +60,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = NvidiaLLMService( - api_key=os.getenv("NVIDIA_API_KEY"), + api_key=os.environ["NVIDIA_API_KEY"], settings=NvidiaLLMService.Settings( model="nvidia/llama-3.3-nemotron-super-49b-v1.5", # Recommended when turning thinking off diff --git a/examples/function-calling/function-calling-ollama.py b/examples/function-calling/function-calling-ollama.py index 87491f3d6..ec6c3948a 100644 --- a/examples/function-calling/function-calling-ollama.py +++ b/examples/function-calling/function-calling-ollama.py @@ -64,10 +64,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), diff --git a/examples/function-calling/function-calling-openai-async-stream.py b/examples/function-calling/function-calling-openai-async-stream.py index a60a5198f..b1d9d724b 100644 --- a/examples/function-calling/function-calling-openai-async-stream.py +++ b/examples/function-calling/function-calling-openai-async-stream.py @@ -107,17 +107,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], enable_async_tool_cancellation=True, settings=OpenAILLMService.Settings( system_instruction=( diff --git a/examples/function-calling/function-calling-openai-async.py b/examples/function-calling/function-calling-openai-async.py index 31c932d5f..47039711f 100644 --- a/examples/function-calling/function-calling-openai-async.py +++ b/examples/function-calling/function-calling-openai-async.py @@ -70,7 +70,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = OpenAISTTService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAISTTService.Settings( model="gpt-4o-transcribe", prompt="Expect words related weather, such as temperature and conditions. And restaurant names.", @@ -78,7 +78,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = OpenAITTSService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAITTSService.Settings( voice="ballad", ), @@ -86,7 +86,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], enable_async_tool_cancellation=True, settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/function-calling/function-calling-openai-responses-async-stream.py b/examples/function-calling/function-calling-openai-responses-async-stream.py index 745c18ae1..4007c21eb 100644 --- a/examples/function-calling/function-calling-openai-responses-async-stream.py +++ b/examples/function-calling/function-calling-openai-responses-async-stream.py @@ -107,17 +107,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], enable_async_tool_cancellation=True, settings=OpenAIResponsesLLMService.Settings( system_instruction=( diff --git a/examples/function-calling/function-calling-openai-responses-async.py b/examples/function-calling/function-calling-openai-responses-async.py index 368f41fe1..f7e6d370b 100644 --- a/examples/function-calling/function-calling-openai-responses-async.py +++ b/examples/function-calling/function-calling-openai-responses-async.py @@ -66,17 +66,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], enable_async_tool_cancellation=True, settings=OpenAIResponsesLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/function-calling/function-calling-openai-responses-http.py b/examples/function-calling/function-calling-openai-responses-http.py index ae38c98f5..e27d5d82b 100644 --- a/examples/function-calling/function-calling-openai-responses-http.py +++ b/examples/function-calling/function-calling-openai-responses-http.py @@ -63,17 +63,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesHttpLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIResponsesHttpLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/function-calling/function-calling-openai-responses-video-http.py b/examples/function-calling/function-calling-openai-responses-video-http.py index ad83aab24..9d9953359 100644 --- a/examples/function-calling/function-calling-openai-responses-video-http.py +++ b/examples/function-calling/function-calling-openai-responses-video-http.py @@ -87,17 +87,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesHttpLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIResponsesHttpLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You are able to describe images from the user camera.", ), diff --git a/examples/function-calling/function-calling-openai-responses-video.py b/examples/function-calling/function-calling-openai-responses-video.py index 9a77af400..3cb159a3f 100644 --- a/examples/function-calling/function-calling-openai-responses-video.py +++ b/examples/function-calling/function-calling-openai-responses-video.py @@ -87,17 +87,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIResponsesLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You are able to describe images from the user camera.", ), diff --git a/examples/function-calling/function-calling-openai-responses.py b/examples/function-calling/function-calling-openai-responses.py index 171e7b36e..97481d01e 100644 --- a/examples/function-calling/function-calling-openai-responses.py +++ b/examples/function-calling/function-calling-openai-responses.py @@ -63,17 +63,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIResponsesLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/function-calling/function-calling-openai-video.py b/examples/function-calling/function-calling-openai-video.py index 3c4fcd2f6..8ca7725b0 100644 --- a/examples/function-calling/function-calling-openai-video.py +++ b/examples/function-calling/function-calling-openai-video.py @@ -87,17 +87,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You are able to describe images from the user camera.", ), diff --git a/examples/function-calling/function-calling-openai.py b/examples/function-calling/function-calling-openai.py index 2b59d7072..2a9779742 100644 --- a/examples/function-calling/function-calling-openai.py +++ b/examples/function-calling/function-calling-openai.py @@ -64,7 +64,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = OpenAISTTService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAISTTService.Settings( model="gpt-4o-transcribe", prompt="Expect words related weather, such as temperature and conditions. And restaurant names.", @@ -72,7 +72,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = OpenAITTSService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAITTSService.Settings( voice="ballad", ), @@ -80,7 +80,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/function-calling/function-calling-openrouter.py b/examples/function-calling/function-calling-openrouter.py index b341ca71c..299846133 100644 --- a/examples/function-calling/function-calling-openrouter.py +++ b/examples/function-calling/function-calling-openrouter.py @@ -60,11 +60,11 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = AzureTTSService( - api_key=os.getenv("AZURE_SPEECH_API_KEY"), - region=os.getenv("AZURE_SPEECH_REGION"), + api_key=os.environ["AZURE_SPEECH_API_KEY"], + region=os.environ["AZURE_SPEECH_REGION"], settings=AzureTTSService.Settings( voice="en-US-JennyNeural", language="en-US", @@ -74,7 +74,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenRouterLLMService( - api_key=os.getenv("OPENROUTER_API_KEY"), + api_key=os.environ["OPENROUTER_API_KEY"], settings=OpenRouterLLMService.Settings( model="openai/gpt-4o-2024-11-20", system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/function-calling/function-calling-perplexity.py b/examples/function-calling/function-calling-perplexity.py index 0fa3ff928..8abbc3618 100644 --- a/examples/function-calling/function-calling-perplexity.py +++ b/examples/function-calling/function-calling-perplexity.py @@ -58,17 +58,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = PerplexityLLMService( - api_key=os.getenv("PERPLEXITY_API_KEY"), + api_key=os.environ["PERPLEXITY_API_KEY"], settings=PerplexityLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/function-calling/function-calling-qwen.py b/examples/function-calling/function-calling-qwen.py index 9a6bf6d8b..0bbb7311d 100644 --- a/examples/function-calling/function-calling-qwen.py +++ b/examples/function-calling/function-calling-qwen.py @@ -60,17 +60,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = QwenLLMService( - api_key=os.getenv("QWEN_API_KEY"), + api_key=os.environ["QWEN_API_KEY"], model="qwen2.5-72b-instruct", settings=QwenLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/function-calling/function-calling-sambanova.py b/examples/function-calling/function-calling-sambanova.py index c1e880bc0..528c4fc47 100644 --- a/examples/function-calling/function-calling-sambanova.py +++ b/examples/function-calling/function-calling-sambanova.py @@ -61,18 +61,18 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = DeepgramSTTService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = SambaNovaLLMService( - api_key=os.getenv("SAMBANOVA_API_KEY"), + api_key=os.environ["SAMBANOVA_API_KEY"], settings=SambaNovaLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/function-calling/function-calling-sarvam.py b/examples/function-calling/function-calling-sarvam.py index bd3fa42fb..830860a9a 100644 --- a/examples/function-calling/function-calling-sarvam.py +++ b/examples/function-calling/function-calling-sarvam.py @@ -64,21 +64,21 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = SarvamSTTService( - api_key=os.getenv("SARVAM_API_KEY"), + api_key=os.environ["SARVAM_API_KEY"], settings=SarvamSTTService.Settings( model="saaras:v3", ), ) tts = SarvamTTSService( - api_key=os.getenv("SARVAM_API_KEY"), + api_key=os.environ["SARVAM_API_KEY"], settings=SarvamTTSService.Settings( model="bulbul:v3", voice="shubh", ), ) llm = SarvamLLMService( - api_key=os.getenv("SARVAM_API_KEY"), + api_key=os.environ["SARVAM_API_KEY"], settings=SarvamLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/function-calling/function-calling-together.py b/examples/function-calling/function-calling-together.py index 78da7fbe9..e785588f4 100644 --- a/examples/function-calling/function-calling-together.py +++ b/examples/function-calling/function-calling-together.py @@ -60,17 +60,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = TogetherLLMService( - api_key=os.getenv("TOGETHER_API_KEY"), + api_key=os.environ["TOGETHER_API_KEY"], settings=TogetherLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/getting-started/01-say-one-thing.py b/examples/getting-started/01-say-one-thing.py index fdfecde6c..1c6e1c703 100644 --- a/examples/getting-started/01-say-one-thing.py +++ b/examples/getting-started/01-say-one-thing.py @@ -36,7 +36,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), diff --git a/examples/getting-started/01a-local-audio.py b/examples/getting-started/01a-local-audio.py index 1e18b4b03..02a489c93 100644 --- a/examples/getting-started/01a-local-audio.py +++ b/examples/getting-started/01a-local-audio.py @@ -28,7 +28,7 @@ async def main(): transport = LocalAudioTransport(LocalAudioTransportParams(audio_out_enabled=True)) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), diff --git a/examples/getting-started/02-llm-say-one-thing.py b/examples/getting-started/02-llm-say-one-thing.py index 74809ed2a..df52772cf 100644 --- a/examples/getting-started/02-llm-say-one-thing.py +++ b/examples/getting-started/02-llm-say-one-thing.py @@ -38,14 +38,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/getting-started/03-still-frame.py b/examples/getting-started/03-still-frame.py index 259072959..61b211499 100644 --- a/examples/getting-started/03-still-frame.py +++ b/examples/getting-started/03-still-frame.py @@ -42,7 +42,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") imagegen = GoogleImageGenService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], ) task = PipelineTask( diff --git a/examples/getting-started/04-sync-speech-and-image.py b/examples/getting-started/04-sync-speech-and-image.py index f0e2ff9c7..c26bb324e 100644 --- a/examples/getting-started/04-sync-speech-and-image.py +++ b/examples/getting-started/04-sync-speech-and-image.py @@ -108,10 +108,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Create an HTTP session for API calls async with aiohttp.ClientSession() as session: - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + llm = OpenAILLMService(api_key=os.environ["OPENAI_API_KEY"]) tts = CartesiaHttpTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaHttpTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), diff --git a/examples/getting-started/05-speaking-state.py b/examples/getting-started/05-speaking-state.py index 36762f6b0..32206d1b0 100644 --- a/examples/getting-started/05-speaking-state.py +++ b/examples/getting-started/05-speaking-state.py @@ -96,17 +96,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/getting-started/06-voice-agent.py b/examples/getting-started/06-voice-agent.py index 057429589..c9fe1cf12 100644 --- a/examples/getting-started/06-voice-agent.py +++ b/examples/getting-started/06-voice-agent.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/getting-started/06a-voice-agent-local.py b/examples/getting-started/06a-voice-agent-local.py index 3d183b405..8864f7aff 100644 --- a/examples/getting-started/06a-voice-agent-local.py +++ b/examples/getting-started/06a-voice-agent-local.py @@ -40,17 +40,17 @@ async def main(): ) ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/getting-started/07-function-calling.py b/examples/getting-started/07-function-calling.py index 085937da9..173f6a6fe 100644 --- a/examples/getting-started/07-function-calling.py +++ b/examples/getting-started/07-function-calling.py @@ -63,17 +63,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/mcp/mcp-multiple-mcp.py b/examples/mcp/mcp-multiple-mcp.py index 7bdfa1343..8d7717291 100644 --- a/examples/mcp/mcp-multiple-mcp.py +++ b/examples/mcp/mcp-multiple-mcp.py @@ -53,10 +53,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -77,7 +77,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): """ llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), + api_key=os.environ["ANTHROPIC_API_KEY"], settings=AnthropicLLMService.Settings( system_instruction=system_prompt, ), diff --git a/examples/mcp/mcp-stdio.py b/examples/mcp/mcp-stdio.py index daae45c0c..a93463ede 100644 --- a/examples/mcp/mcp-stdio.py +++ b/examples/mcp/mcp-stdio.py @@ -49,10 +49,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -71,7 +71,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): """ llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), + api_key=os.environ["ANTHROPIC_API_KEY"], settings=AnthropicLLMService.Settings( system_instruction=system_prompt, ), diff --git a/examples/mcp/mcp-streamable-http-gemini-live.py b/examples/mcp/mcp-streamable-http-gemini-live.py index 8b824454f..213667e7a 100644 --- a/examples/mcp/mcp-streamable-http-gemini-live.py +++ b/examples/mcp/mcp-streamable-http-gemini-live.py @@ -23,8 +23,6 @@ from pipecat.processors.aggregators.llm_response_universal import ( ) from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService from pipecat.services.mcp_service import MCPClient from pipecat.transports.base_transport import BaseTransport, TransportParams @@ -54,15 +52,6 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - settings=CartesiaTTSService.Settings( - voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ), - ) - system = f""" You are a helpful LLM in a voice call. Your goal is to answer questions about the user's GitHub repositories and account. @@ -85,7 +74,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): tools = await mcp.get_tools_schema() llm = GeminiLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], system_instruction=system, tools=tools, ) diff --git a/examples/mcp/mcp-streamable-http.py b/examples/mcp/mcp-streamable-http.py index 59860d307..b4823c649 100644 --- a/examples/mcp/mcp-streamable-http.py +++ b/examples/mcp/mcp-streamable-http.py @@ -54,10 +54,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -73,7 +73,7 @@ Just respond with short sentences when you are carrying out tool calls. """ llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( system_instruction=system_prompt, ), diff --git a/examples/observability/observability-observer.py b/examples/observability/observability-observer.py index 457ba7bb4..4a1bbbbfa 100644 --- a/examples/observability/observability-observer.py +++ b/examples/observability/observability-observer.py @@ -100,17 +100,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/observability/observability-sentry-metrics.py b/examples/observability/observability-sentry-metrics.py index 3d4146150..9a8f8d553 100644 --- a/examples/observability/observability-sentry-metrics.py +++ b/examples/observability/observability-sentry-metrics.py @@ -60,12 +60,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) stt = DeepgramSTTService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], metrics=SentryMetrics(), ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -73,7 +73,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], metrics=SentryMetrics(), settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/persistent-context/persistent-context-anthropic.py b/examples/persistent-context/persistent-context-anthropic.py index 791f00440..cdddc9080 100644 --- a/examples/persistent-context/persistent-context-anthropic.py +++ b/examples/persistent-context/persistent-context-anthropic.py @@ -170,17 +170,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), + api_key=os.environ["ANTHROPIC_API_KEY"], settings=AnthropicLLMService.Settings( system_instruction=system_instruction, ), diff --git a/examples/persistent-context/persistent-context-aws-nova-sonic.py b/examples/persistent-context/persistent-context-aws-nova-sonic.py index 62b9e7eab..8a4b7ba47 100644 --- a/examples/persistent-context/persistent-context-aws-nova-sonic.py +++ b/examples/persistent-context/persistent-context-aws-nova-sonic.py @@ -87,7 +87,9 @@ async def save_conversation(params: FunctionCallParams): # the simplest thing to do is to pop messages until the last one is an assistant # response while messages and not ( - messages[-1].get("role") == "assistant" and "content" in messages[-1] + isinstance(messages[-1], dict) + and messages[-1].get("role") == "assistant" + and "content" in messages[-1] ): messages.pop() if messages: # we never expect this to be empty @@ -125,6 +127,7 @@ async def load_conversation(params: FunctionCallParams): } ) params.context.set_messages(messages) + assert isinstance(params.llm, AWSNovaSonicLLMService) await params.llm.reset_conversation() # await params.llm.trigger_assistant_response() except Exception as e: @@ -219,9 +222,9 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = AWSNovaSonicLLMService( - secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), - access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), - region=os.getenv("AWS_REGION"), # as of 2025-05-06, us-east-1 is the only supported region + secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"], + access_key_id=os.environ["AWS_ACCESS_KEY_ID"], + region=os.environ["AWS_REGION"], # as of 2025-05-06, us-east-1 is the only supported region settings=AWSNovaSonicLLMService.Settings( voice="tiffany", # matthew, tiffany, amy system_instruction=system_instruction, diff --git a/examples/persistent-context/persistent-context-gemini.py b/examples/persistent-context/persistent-context-gemini.py index 2ec3cd0bb..2ae43bf87 100644 --- a/examples/persistent-context/persistent-context-gemini.py +++ b/examples/persistent-context/persistent-context-gemini.py @@ -243,17 +243,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], system_instruction=system_instruction, ) diff --git a/examples/persistent-context/persistent-context-grok-realtime.py b/examples/persistent-context/persistent-context-grok-realtime.py index 4a990d24e..e21dbf1c3 100644 --- a/examples/persistent-context/persistent-context-grok-realtime.py +++ b/examples/persistent-context/persistent-context-grok-realtime.py @@ -192,7 +192,7 @@ Remember, your responses should be short - just one or two sentences usually.""" ) llm = GrokRealtimeLLMService( - api_key=os.getenv("XAI_API_KEY"), + api_key=os.environ["XAI_API_KEY"], session_properties=session_properties, ) diff --git a/examples/persistent-context/persistent-context-openai-realtime.py b/examples/persistent-context/persistent-context-openai-realtime.py index 0a7a888a8..067e1f73f 100644 --- a/examples/persistent-context/persistent-context-openai-realtime.py +++ b/examples/persistent-context/persistent-context-openai-realtime.py @@ -93,6 +93,7 @@ async def load_conversation(params: FunctionCallParams): try: with open(filename) as file: params.context.set_messages(json.load(file)) + assert isinstance(params.llm, OpenAIRealtimeLLMService) await params.llm.reset_conversation() # NOTE: we manually create a response here rather than relying # on the function callback to trigger one since we've reset the @@ -171,10 +172,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) llm = OpenAIRealtimeLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIRealtimeLLMService.Settings( system_instruction="""Your knowledge cutoff is 2023-10. You are a helpful and friendly AI. diff --git a/examples/persistent-context/persistent-context-openai-responses-http.py b/examples/persistent-context/persistent-context-openai-responses-http.py index d77513049..362fd86f6 100644 --- a/examples/persistent-context/persistent-context-openai-responses-http.py +++ b/examples/persistent-context/persistent-context-openai-responses-http.py @@ -171,17 +171,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesHttpLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIResponsesHttpLLMService.Settings( system_instruction=system_instruction, ), diff --git a/examples/persistent-context/persistent-context-openai-responses.py b/examples/persistent-context/persistent-context-openai-responses.py index c89fe3ff1..9c3f33c94 100644 --- a/examples/persistent-context/persistent-context-openai-responses.py +++ b/examples/persistent-context/persistent-context-openai-responses.py @@ -171,17 +171,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIResponsesLLMService.Settings( system_instruction=system_instruction, ), diff --git a/examples/persistent-context/persistent-context-openai.py b/examples/persistent-context/persistent-context-openai.py index 22e160102..fa29ad5c2 100644 --- a/examples/persistent-context/persistent-context-openai.py +++ b/examples/persistent-context/persistent-context-openai.py @@ -171,17 +171,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], system_instruction=system_instruction, ) diff --git a/examples/rag/rag-gemini-grounding-metadata.py b/examples/rag/rag-gemini-grounding-metadata.py index 5b706d6c9..cddefe591 100644 --- a/examples/rag/rag-gemini-grounding-metadata.py +++ b/examples/rag/rag-gemini-grounding-metadata.py @@ -95,10 +95,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -106,7 +106,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Initialize the Gemini Multimodal Live model llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( system_instruction=system_instruction, ), diff --git a/examples/rag/rag-gemini.py b/examples/rag/rag-gemini.py index 0f9a8ddb2..1567ad051 100644 --- a/examples/rag/rag-gemini.py +++ b/examples/rag/rag-gemini.py @@ -52,7 +52,7 @@ import os import time from dotenv import load_dotenv -from google import genai +from google import genai # pyright: ignore[reportAttributeAccessIssue] from loguru import logger from pipecat.adapters.schemas.function_schema import FunctionSchema @@ -179,10 +179,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="f9836c6e-a0bd-460e-9d3c-f7299fa60f94", # Southern Lady ), @@ -197,7 +197,7 @@ Your response will be turned into speech so use only simple words and punctuatio """ llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( model=VOICE_MODEL, system_instruction=system_prompt, diff --git a/examples/rag/rag-mem0.py b/examples/rag/rag-mem0.py index c7c0f13bd..1cbf97d14 100644 --- a/examples/rag/rag-mem0.py +++ b/examples/rag/rag-mem0.py @@ -133,11 +133,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) # Initialize text-to-speech service tts = ElevenLabsTTSService( - api_key=os.getenv("ELEVENLABS_API_KEY"), + api_key=os.environ["ELEVENLABS_API_KEY"], settings=ElevenLabsTTSService.Settings( voice="pNInz6obpgDQGcFmaJgB", ), @@ -196,7 +196,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Initialize LLM service llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="""You are a personal assistant. You can remember things about the person you are talking to. Some Guidelines: diff --git a/examples/realtime/realtime-aws-nova-sonic.py b/examples/realtime/realtime-aws-nova-sonic.py index f3a1a0737..5e6a2037c 100644 --- a/examples/realtime/realtime-aws-nova-sonic.py +++ b/examples/realtime/realtime-aws-nova-sonic.py @@ -116,8 +116,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Create the AWS Nova Sonic LLM service llm = AWSNovaSonicLLMService( - secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), - access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), + secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"], + access_key_id=os.environ["AWS_ACCESS_KEY_ID"], # as of 2025-12-09, these are the supported regions: # - Nova 2 Sonic (the default model): # - us-east-1 @@ -126,7 +126,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # - Nova Sonic (the older model): # - us-east-1 # - ap-northeast-1 - region=os.getenv("AWS_REGION"), + region=os.environ["AWS_REGION"], session_token=os.getenv("AWS_SESSION_TOKEN"), settings=AWSNovaSonicLLMService.Settings( voice="tiffany", diff --git a/examples/realtime/realtime-azure.py b/examples/realtime/realtime-azure.py index c57602f6f..c4ff80dcb 100644 --- a/examples/realtime/realtime-azure.py +++ b/examples/realtime/realtime-azure.py @@ -112,8 +112,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") llm = AzureRealtimeLLMService( - api_key=os.getenv("AZURE_REALTIME_API_KEY"), - base_url=os.getenv("AZURE_REALTIME_BASE_URL"), + api_key=os.environ["AZURE_REALTIME_API_KEY"], + base_url=os.environ["AZURE_REALTIME_BASE_URL"], settings=AzureRealtimeLLMService.Settings( system_instruction="""You are a helpful and friendly AI. diff --git a/examples/realtime/realtime-gemini-live-files-api.py b/examples/realtime/realtime-gemini-live-files-api.py index 1871bb5bf..99f35d52a 100644 --- a/examples/realtime/realtime-gemini-live-files-api.py +++ b/examples/realtime/realtime-gemini-live-files-api.py @@ -104,7 +104,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Initialize Gemini service with File API support llm = GeminiLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GeminiLiveLLMService.Settings( system_instruction=system_instruction, voice="Charon", # Aoede, Charon, Fenrir, Kore, Puck diff --git a/examples/realtime/realtime-gemini-live-function-calling.py b/examples/realtime/realtime-gemini-live-function-calling.py index a370fdf57..d038a5c4f 100644 --- a/examples/realtime/realtime-gemini-live-function-calling.py +++ b/examples/realtime/realtime-gemini-live-function-calling.py @@ -114,7 +114,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = GeminiLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GeminiLiveLLMService.Settings( system_instruction=system_instruction, ), diff --git a/examples/realtime/realtime-gemini-live-google-search.py b/examples/realtime/realtime-gemini-live-google-search.py index 7822bb085..6b132cef1 100644 --- a/examples/realtime/realtime-gemini-live-google-search.py +++ b/examples/realtime/realtime-gemini-live-google-search.py @@ -67,7 +67,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Initialize the Gemini Multimodal Live model llm = GeminiLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GeminiLiveLLMService.Settings( voice="Puck", # Aoede, Charon, Fenrir, Kore, Puck system_instruction=system_instruction, diff --git a/examples/realtime/realtime-gemini-live-graceful-end.py b/examples/realtime/realtime-gemini-live-graceful-end.py index 7eca04fb3..d65414022 100644 --- a/examples/realtime/realtime-gemini-live-graceful-end.py +++ b/examples/realtime/realtime-gemini-live-graceful-end.py @@ -133,7 +133,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = GeminiLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GeminiLiveLLMService.Settings( system_instruction=system_instruction, ), diff --git a/examples/realtime/realtime-gemini-live-grounding-metadata.py b/examples/realtime/realtime-gemini-live-grounding-metadata.py index 31fd7d832..e78d5cffa 100644 --- a/examples/realtime/realtime-gemini-live-grounding-metadata.py +++ b/examples/realtime/realtime-gemini-live-grounding-metadata.py @@ -75,7 +75,8 @@ class GroundingMetadataProcessor(FrameProcessor): if isinstance(frame, LLMSearchResponseFrame): self._grounding_count += 1 logger.info(f"\n\n🔍 GROUNDING METADATA RECEIVED #{self._grounding_count}\n") - logger.info(f"📝 Search Result Text: {frame.search_result[:200]}...") + if frame.search_result: + logger.info(f"📝 Search Result Text: {frame.search_result[:200]}...") if frame.rendered_content: logger.info(f"🔗 Rendered Content: {frame.rendered_content}") @@ -101,7 +102,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = GeminiLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GeminiLiveLLMService.Settings( system_instruction=SYSTEM_INSTRUCTION, voice="Charon", # Aoede, Charon, Fenrir, Kore, Puck @@ -111,16 +112,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Create a processor to capture grounding metadata grounding_processor = GroundingMetadataProcessor() - - messages = [ - { - "role": "user", - "content": "Please introduce yourself and let me know that you can help with current information by searching the web. Ask me what current information I'd like to know about.", - }, - ] - # Set up conversation context and management - context = LLMContext(messages) + context = LLMContext() # Server-side VAD is enabled by default; no local VAD is added. user_aggregator, assistant_aggregator = LLMContextAggregatorPair(context) @@ -144,6 +137,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): async def on_client_connected(transport, client): logger.info(f"Client connected") # Kick off the conversation. + context.add_message( + { + "role": "developer", + "content": "Please introduce yourself and let me know that you can help with current information by searching the web. Ask me what current information I'd like to know about.", + } + ) await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") diff --git a/examples/realtime/realtime-gemini-live-local-vad.py b/examples/realtime/realtime-gemini-live-local-vad.py index 7a73742e9..c7580f0c6 100644 --- a/examples/realtime/realtime-gemini-live-local-vad.py +++ b/examples/realtime/realtime-gemini-live-local-vad.py @@ -54,7 +54,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") llm = GeminiLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GeminiLiveLLMService.Settings( voice="Aoede", # Puck, Charon, Kore, Fenrir, Aoede vad=GeminiVADParams(disabled=True), diff --git a/examples/realtime/realtime-gemini-live-vertex-function-calling.py b/examples/realtime/realtime-gemini-live-vertex-function-calling.py index 68add5bc9..2c0861006 100644 --- a/examples/realtime/realtime-gemini-live-vertex-function-calling.py +++ b/examples/realtime/realtime-gemini-live-vertex-function-calling.py @@ -110,8 +110,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = GeminiLiveVertexLLMService( credentials=os.getenv("GOOGLE_VERTEX_TEST_CREDENTIALS"), - project_id=os.getenv("GOOGLE_CLOUD_PROJECT_ID"), - location=os.getenv("GOOGLE_CLOUD_LOCATION"), + project_id=os.environ["GOOGLE_CLOUD_PROJECT_ID"], + location=os.environ["GOOGLE_CLOUD_LOCATION"], settings=GeminiLiveVertexLLMService.Settings( system_instruction=system_instruction, voice="Puck", # Aoede, Charon, Fenrir, Kore, Puck diff --git a/examples/realtime/realtime-gemini-live-video.py b/examples/realtime/realtime-gemini-live-video.py index 269f22718..4319b534a 100644 --- a/examples/realtime/realtime-gemini-live-video.py +++ b/examples/realtime/realtime-gemini-live-video.py @@ -47,7 +47,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = GeminiLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GeminiLiveLLMService.Settings( voice="Aoede", # Puck, Charon, Kore, Fenrir, Aoede # system_instruction="Talk like a pirate." diff --git a/examples/realtime/realtime-gemini-live.py b/examples/realtime/realtime-gemini-live.py index dc9e8e55a..04fa9c625 100644 --- a/examples/realtime/realtime-gemini-live.py +++ b/examples/realtime/realtime-gemini-live.py @@ -52,7 +52,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") llm = GeminiLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GeminiLiveLLMService.Settings( voice="Aoede", # Puck, Charon, Kore, Fenrir, Aoede # system_instruction="Talk like a pirate." diff --git a/examples/realtime/realtime-grok.py b/examples/realtime/realtime-grok.py index 3f1543871..1e122865a 100644 --- a/examples/realtime/realtime-grok.py +++ b/examples/realtime/realtime-grok.py @@ -179,7 +179,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Create the Grok Realtime LLM service llm = GrokRealtimeLLMService( - api_key=os.getenv("XAI_API_KEY"), + api_key=os.environ["XAI_API_KEY"], settings=GrokRealtimeLLMService.Settings( system_instruction="""You are a helpful and friendly AI assistant powered by Grok. diff --git a/examples/realtime/realtime-inworld.py b/examples/realtime/realtime-inworld.py index 67bed6df9..76defbc24 100644 --- a/examples/realtime/realtime-inworld.py +++ b/examples/realtime/realtime-inworld.py @@ -84,7 +84,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # llm_model can be any supported model or an Inworld Router. # See: https://docs.inworld.ai/router/introduction llm = InworldRealtimeLLMService( - api_key=os.getenv("INWORLD_API_KEY"), + api_key=os.environ["INWORLD_API_KEY"], llm_model="xai/grok-4-1-fast-non-reasoning", voice="Sarah", settings=InworldRealtimeLLMService.Settings( diff --git a/examples/realtime/realtime-openai-live-video.py b/examples/realtime/realtime-openai-live-video.py index fdf7c04e8..b35aa2648 100644 --- a/examples/realtime/realtime-openai-live-video.py +++ b/examples/realtime/realtime-openai-live-video.py @@ -62,7 +62,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") llm = OpenAIRealtimeLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIRealtimeLLMService.Settings( system_instruction="""You are a helpful and friendly AI. @@ -133,8 +133,8 @@ Remember, your responses should be short. Just one or two sentences, usually. Re async def on_client_connected(transport, client): logger.info(f"Client connected: {client}") - await maybe_capture_participant_camera(transport, client, framerate=0.5) - await maybe_capture_participant_screen(transport, client, framerate=0.5) + await maybe_capture_participant_camera(transport, client, framerate=1) + await maybe_capture_participant_screen(transport, client, framerate=1) await task.queue_frames([LLMRunFrame()]) diff --git a/examples/realtime/realtime-openai-text.py b/examples/realtime/realtime-openai-text.py index 4147ca4c3..f36a3c773 100644 --- a/examples/realtime/realtime-openai-text.py +++ b/examples/realtime/realtime-openai-text.py @@ -117,7 +117,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") llm = OpenAIRealtimeLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIRealtimeLLMSettings( system_instruction="""You are a helpful and friendly AI. @@ -156,7 +156,7 @@ Remember, your responses should be short. Just one or two sentences, usually. Re ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), diff --git a/examples/realtime/realtime-openai.py b/examples/realtime/realtime-openai.py index 9c47bac77..6f347ed9b 100644 --- a/examples/realtime/realtime-openai.py +++ b/examples/realtime/realtime-openai.py @@ -136,7 +136,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") llm = OpenAIRealtimeLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIRealtimeLLMService.Settings( system_instruction="""You are a helpful and friendly AI. diff --git a/examples/realtime/realtime-ultravox-text.py b/examples/realtime/realtime-ultravox-text.py index 8b876048a..a144af919 100644 --- a/examples/realtime/realtime-ultravox-text.py +++ b/examples/realtime/realtime-ultravox-text.py @@ -168,7 +168,7 @@ There is also a secret menu that changes daily. If the user asks about it, use t llm = UltravoxRealtimeLLMService( params=OneShotInputParams( - api_key=os.getenv("ULTRAVOX_API_KEY"), + api_key=os.environ["ULTRAVOX_API_KEY"], system_prompt=system_prompt, temperature=0.3, max_duration=datetime.timedelta(minutes=3), diff --git a/examples/realtime/realtime-ultravox.py b/examples/realtime/realtime-ultravox.py index 7aaacdeaf..ed46aba2f 100644 --- a/examples/realtime/realtime-ultravox.py +++ b/examples/realtime/realtime-ultravox.py @@ -166,7 +166,7 @@ There is also a secret menu that changes daily. If the user asks about it, use t llm = UltravoxRealtimeLLMService( params=OneShotInputParams( - api_key=os.getenv("ULTRAVOX_API_KEY"), + api_key=os.environ["ULTRAVOX_API_KEY"], system_prompt=system_prompt, temperature=0.3, max_duration=datetime.timedelta(minutes=3), diff --git a/examples/thinking/thinking-anthropic.py b/examples/thinking/thinking-anthropic.py index 0cd768925..fe64aa3f3 100644 --- a/examples/thinking/thinking-anthropic.py +++ b/examples/thinking/thinking-anthropic.py @@ -52,17 +52,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), + api_key=os.environ["ANTHROPIC_API_KEY"], settings=AnthropicLLMService.Settings( thinking=AnthropicLLMService.ThinkingConfig( type="enabled", diff --git a/examples/thinking/thinking-functions-anthropic.py b/examples/thinking/thinking-functions-anthropic.py index b62a96452..f69b271fb 100644 --- a/examples/thinking/thinking-functions-anthropic.py +++ b/examples/thinking/thinking-functions-anthropic.py @@ -73,17 +73,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), + api_key=os.environ["ANTHROPIC_API_KEY"], settings=AnthropicLLMService.Settings( thinking=AnthropicLLMService.ThinkingConfig( type="enabled", diff --git a/examples/thinking/thinking-functions-google.py b/examples/thinking/thinking-functions-google.py index e06a0195b..680f2c70b 100644 --- a/examples/thinking/thinking-functions-google.py +++ b/examples/thinking/thinking-functions-google.py @@ -73,17 +73,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], # model="gemini-3-pro-preview", # A more powerful reasoning model, but slower settings=GoogleLLMService.Settings( thinking=GoogleLLMService.ThinkingConfig( diff --git a/examples/thinking/thinking-google.py b/examples/thinking/thinking-google.py index 94d3d28eb..12de4a313 100644 --- a/examples/thinking/thinking-google.py +++ b/examples/thinking/thinking-google.py @@ -52,17 +52,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], # model="gemini-3-pro-preview", # A more powerful reasoning model, but slower settings=GoogleLLMService.Settings( thinking=GoogleLLMService.ThinkingConfig( diff --git a/examples/transcription/transcription-assemblyai.py b/examples/transcription/transcription-assemblyai.py index f50f63380..f9db2c85e 100644 --- a/examples/transcription/transcription-assemblyai.py +++ b/examples/transcription/transcription-assemblyai.py @@ -48,7 +48,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = AssemblyAISTTService( - api_key=os.getenv("ASSEMBLYAI_API_KEY"), + api_key=os.environ["ASSEMBLYAI_API_KEY"], settings=AssemblyAISTTService.Settings( model="u3-rt-pro", ), diff --git a/examples/transcription/transcription-azure.py b/examples/transcription/transcription-azure.py index 301c6effd..1a551f745 100644 --- a/examples/transcription/transcription-azure.py +++ b/examples/transcription/transcription-azure.py @@ -54,7 +54,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = AzureSTTService( - api_key=os.getenv("AZURE_SPEECH_API_KEY"), + api_key=os.environ["AZURE_SPEECH_API_KEY"], region=os.getenv("AZURE_SPEECH_REGION"), ) diff --git a/examples/transcription/transcription-cartesia.py b/examples/transcription/transcription-cartesia.py index d3b83abb0..f86e5fa37 100644 --- a/examples/transcription/transcription-cartesia.py +++ b/examples/transcription/transcription-cartesia.py @@ -47,7 +47,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = CartesiaSTTService(api_key=os.getenv("CARTESIA_API_KEY")) + stt = CartesiaSTTService(api_key=os.environ["CARTESIA_API_KEY"]) tl = TranscriptionLogger() diff --git a/examples/transcription/transcription-deepgram.py b/examples/transcription/transcription-deepgram.py index 04246d237..92fe076e3 100644 --- a/examples/transcription/transcription-deepgram.py +++ b/examples/transcription/transcription-deepgram.py @@ -48,7 +48,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = DeepgramSTTService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], settings=DeepgramSTTService.Settings( language=Language.EN, ), diff --git a/examples/transcription/transcription-elevenlabs.py b/examples/transcription/transcription-elevenlabs.py index dbfd32ec6..548a12de2 100644 --- a/examples/transcription/transcription-elevenlabs.py +++ b/examples/transcription/transcription-elevenlabs.py @@ -49,7 +49,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = ElevenLabsRealtimeSTTService(api_key=os.getenv("ELEVENLABS_API_KEY")) + stt = ElevenLabsRealtimeSTTService(api_key=os.environ["ELEVENLABS_API_KEY"]) tl = TranscriptionLogger() vad_processor = VADProcessor(vad_analyzer=SileroVADAnalyzer()) diff --git a/examples/transcription/transcription-gladia-translation.py b/examples/transcription/transcription-gladia-translation.py index e6557cd15..477c73b23 100644 --- a/examples/transcription/transcription-gladia-translation.py +++ b/examples/transcription/transcription-gladia-translation.py @@ -55,9 +55,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") + region = os.getenv("GLADIA_REGION", "us-west") + assert region in ("us-west", "eu-west"), f"Invalid GLADIA_REGION: {region}" + stt = GladiaSTTService( - api_key=os.getenv("GLADIA_API_KEY"), - region=os.getenv("GLADIA_REGION"), + api_key=os.environ["GLADIA_API_KEY"], + region=region, settings=GladiaSTTService.Settings( language_config=LanguageConfig( languages=[Language.EN], diff --git a/examples/transcription/transcription-gladia.py b/examples/transcription/transcription-gladia.py index e98d4caf0..cd1152972 100644 --- a/examples/transcription/transcription-gladia.py +++ b/examples/transcription/transcription-gladia.py @@ -47,9 +47,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") + region = os.getenv("GLADIA_REGION", "us-west") + assert region in ("us-west", "eu-west"), f"Invalid GLADIA_REGION: {region}" + stt = GladiaSTTService( - api_key=os.getenv("GLADIA_API_KEY"), - region=os.getenv("GLADIA_REGION"), + api_key=os.environ["GLADIA_API_KEY"], + region=region, # settings=GladiaSTTSettings( # language_config=LanguageConfig( # languages=[Language.FR], diff --git a/examples/transcription/transcription-google-llm.py b/examples/transcription/transcription-google-llm.py index c22ed85a1..909711ecd 100644 --- a/examples/transcription/transcription-google-llm.py +++ b/examples/transcription/transcription-google-llm.py @@ -289,7 +289,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -301,7 +301,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): model="gemini-2.5-flash", system_instruction=conversation_system_message, ), - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], # we can give the GoogleLLMService a system instruction to use directly # in the GenerativeModel constructor. Let's do that rather than put # our system message in the messages list. @@ -313,7 +313,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): model="gemini-2.5-flash", system_instruction=transcriber_system_message, ), - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], ) messages = [ diff --git a/examples/transcription/transcription-gradium.py b/examples/transcription/transcription-gradium.py index 59140466e..0b7a4412d 100644 --- a/examples/transcription/transcription-gradium.py +++ b/examples/transcription/transcription-gradium.py @@ -50,7 +50,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = GradiumSTTService( - api_key=os.getenv("GRADIUM_API_KEY"), + api_key=os.environ["GRADIUM_API_KEY"], api_endpoint_base_url="wss://us.api.gradium.ai/api/speech/asr", settings=GradiumSTTService.Settings( language=Language.EN, diff --git a/examples/transcription/transcription-mistral.py b/examples/transcription/transcription-mistral.py index b040b457c..fa471d2e6 100644 --- a/examples/transcription/transcription-mistral.py +++ b/examples/transcription/transcription-mistral.py @@ -54,7 +54,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = MistralSTTService( - api_key=os.getenv("MISTRAL_API_KEY"), + api_key=os.environ["MISTRAL_API_KEY"], ) tl = TranscriptionLogger() diff --git a/examples/transcription/transcription-openai.py b/examples/transcription/transcription-openai.py index cbbf0e13d..5f3072177 100644 --- a/examples/transcription/transcription-openai.py +++ b/examples/transcription/transcription-openai.py @@ -50,7 +50,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = OpenAIRealtimeSTTService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIRealtimeSTTService.Settings( model="gpt-4o-transcribe", prompt="Expect words related to dogs, such as breed names.", diff --git a/examples/transcription/transcription-soniox.py b/examples/transcription/transcription-soniox.py index 9476e9441..b912fa04c 100644 --- a/examples/transcription/transcription-soniox.py +++ b/examples/transcription/transcription-soniox.py @@ -54,7 +54,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = SonioxSTTService( - api_key=os.getenv("SONIOX_API_KEY"), + api_key=os.environ["SONIOX_API_KEY"], ) tl = TranscriptionLogger() diff --git a/examples/transcription/transcription-speechmatics.py b/examples/transcription/transcription-speechmatics.py index 2df30f67b..2134c8c97 100644 --- a/examples/transcription/transcription-speechmatics.py +++ b/examples/transcription/transcription-speechmatics.py @@ -64,7 +64,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = SpeechmaticsSTTService( - api_key=os.getenv("SPEECHMATICS_API_KEY"), + api_key=os.environ["SPEECHMATICS_API_KEY"], settings=SpeechmaticsSTTService.Settings( language=Language.EN, speaker_active_format="<{speaker_id}>{text}", diff --git a/examples/transcription/transcription-xai.py b/examples/transcription/transcription-xai.py new file mode 100644 index 000000000..0cc667f9a --- /dev/null +++ b/examples/transcription/transcription-xai.py @@ -0,0 +1,86 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.frames.frames import Frame, TranscriptionFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineTask +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.xai.stt import XAISTTService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + + +class TranscriptionLogger(FrameProcessor): + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, TranscriptionFrame): + print(f"Transcription: {frame.text}") + + # Push all frames through + await self.push_frame(frame, direction) + + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = XAISTTService( + api_key=os.getenv("XAI_API_KEY"), + ) + + tl = TranscriptionLogger() + + pipeline = Pipeline([transport.input(), stt, tl]) + + task = PipelineTask( + pipeline, + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/transports/transports-daily.py b/examples/transports/transports-daily.py index c6695e2f8..09deda2dd 100644 --- a/examples/transports/transports-daily.py +++ b/examples/transports/transports-daily.py @@ -49,14 +49,14 @@ async def main(): ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/transports/transports-livekit.py b/examples/transports/transports-livekit.py index b2ec6b49c..8bb147c04 100644 --- a/examples/transports/transports-livekit.py +++ b/examples/transports/transports-livekit.py @@ -53,17 +53,17 @@ async def main(): ), ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), diff --git a/examples/transports/transports-small-webrtc.py b/examples/transports/transports-small-webrtc.py index 18403940b..2cb6d4bb0 100644 --- a/examples/transports/transports-small-webrtc.py +++ b/examples/transports/transports-small-webrtc.py @@ -62,17 +62,17 @@ async def run_example(webrtc_connection: SmallWebRTCConnection): ), ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/turn-management/turn-management-detect-user-idle.py b/examples/turn-management/turn-management-detect-user-idle.py index a159f0c4e..8391e3908 100644 --- a/examples/turn-management/turn-management-detect-user-idle.py +++ b/examples/turn-management/turn-management-detect-user-idle.py @@ -11,6 +11,7 @@ import os from dotenv import load_dotenv from loguru import logger +from pipecat.adapters.base_llm_adapter import LLMContextMessage from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -59,7 +60,7 @@ class IdleHandler: if self._retry_count == 1: # First attempt: Add a gentle prompt to the conversation - message = { + message: LLMContextMessage = { "role": "developer", "content": "The user has been quiet. Politely and briefly ask if they're still there.", } @@ -111,17 +112,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/turn-management/turn-management-filter-incomplete-turns.py b/examples/turn-management/turn-management-filter-incomplete-turns.py index fca89f66c..81273aaaa 100644 --- a/examples/turn-management/turn-management-filter-incomplete-turns.py +++ b/examples/turn-management/turn-management-filter-incomplete-turns.py @@ -66,17 +66,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), diff --git a/examples/turn-management/turn-management-interruption-config.py b/examples/turn-management/turn-management-interruption-config.py index d625910df..ee19075df 100644 --- a/examples/turn-management/turn-management-interruption-config.py +++ b/examples/turn-management/turn-management-interruption-config.py @@ -55,17 +55,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/turn-management/turn-management-smart-turn-local-coreml.py b/examples/turn-management/turn-management-smart-turn-local-coreml.py index 3042ceb5a..68e831bba 100644 --- a/examples/turn-management/turn-management-smart-turn-local-coreml.py +++ b/examples/turn-management/turn-management-smart-turn-local-coreml.py @@ -49,7 +49,7 @@ load_dotenv(override=True) # Then set the env variable: # export LOCAL_SMART_TURN_MODEL_PATH=./smart-turn # or add it to your .env file -smart_turn_model_path = os.getenv("LOCAL_SMART_TURN_MODEL_PATH") +smart_turn_model_path = os.environ["LOCAL_SMART_TURN_MODEL_PATH"] # We use lambdas to defer transport parameter creation until the transport # type is selected at runtime. @@ -72,17 +72,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/turn-management/turn-management-smart-turn-local.py b/examples/turn-management/turn-management-smart-turn-local.py index 7a73a4324..5509f0264 100644 --- a/examples/turn-management/turn-management-smart-turn-local.py +++ b/examples/turn-management/turn-management-smart-turn-local.py @@ -54,17 +54,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/turn-management/turn-management-turn-tracking-observer.py b/examples/turn-management/turn-management-turn-tracking-observer.py index 14d63d40b..86f061906 100644 --- a/examples/turn-management/turn-management-turn-tracking-observer.py +++ b/examples/turn-management/turn-management-turn-tracking-observer.py @@ -69,17 +69,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/turn-management/turn-management-user-assistant-turns.py b/examples/turn-management/turn-management-user-assistant-turns.py index 380c4866d..879eb1cdc 100644 --- a/examples/turn-management/turn-management-user-assistant-turns.py +++ b/examples/turn-management/turn-management-user-assistant-turns.py @@ -118,17 +118,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/turn-management/turn-management-user-mute-strategy.py b/examples/turn-management/turn-management-user-mute-strategy.py index 6e826f81d..0f9b24a8c 100644 --- a/examples/turn-management/turn-management-user-mute-strategy.py +++ b/examples/turn-management/turn-management-user-mute-strategy.py @@ -69,17 +69,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = DeepgramTTSService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], settings=DeepgramTTSService.Settings( voice="aura-2-helena-en", ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant who can check the weather. Always check the weather when a location is mentioned. Respond concisely and naturally. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points.", ), diff --git a/examples/update-settings/llm/llm-anthropic.py b/examples/update-settings/llm/llm-anthropic.py index 67ba6e9e9..885981ddd 100644 --- a/examples/update-settings/llm/llm-anthropic.py +++ b/examples/update-settings/llm/llm-anthropic.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), + api_key=os.environ["ANTHROPIC_API_KEY"], settings=AnthropicLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-aws-bedrock.py b/examples/update-settings/llm/llm-aws-bedrock.py index 168bb6bf0..6facff628 100644 --- a/examples/update-settings/llm/llm-aws-bedrock.py +++ b/examples/update-settings/llm/llm-aws-bedrock.py @@ -50,10 +50,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), diff --git a/examples/update-settings/llm/llm-aws-nova-sonic.py b/examples/update-settings/llm/llm-aws-nova-sonic.py index 99fa5cd7f..19ca3c805 100644 --- a/examples/update-settings/llm/llm-aws-nova-sonic.py +++ b/examples/update-settings/llm/llm-aws-nova-sonic.py @@ -49,9 +49,9 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") llm = AWSNovaSonicLLMService( - secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), - access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), - region=os.getenv("AWS_REGION"), + secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"], + access_key_id=os.environ["AWS_ACCESS_KEY_ID"], + region=os.environ["AWS_REGION"], settings=AWSNovaSonicLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-azure-realtime.py b/examples/update-settings/llm/llm-azure-realtime.py index fde633912..3d724bd53 100644 --- a/examples/update-settings/llm/llm-azure-realtime.py +++ b/examples/update-settings/llm/llm-azure-realtime.py @@ -10,6 +10,7 @@ import os from dotenv import load_dotenv from loguru import logger +from pipecat.adapters.base_llm_adapter import LLMContextMessage from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame from pipecat.pipeline.pipeline import Pipeline @@ -51,11 +52,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") llm = AzureRealtimeLLMService( - api_key=os.getenv("AZURE_REALTIME_API_KEY"), - base_url=os.getenv("AZURE_REALTIME_BASE_URL"), + api_key=os.environ["AZURE_REALTIME_API_KEY"], + base_url=os.environ["AZURE_REALTIME_BASE_URL"], ) - messages = [ + messages: list[LLMContextMessage] = [ { "role": "system", "content": "You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/update-settings/llm/llm-azure.py b/examples/update-settings/llm/llm-azure.py index 60d151484..3f9f5da57 100644 --- a/examples/update-settings/llm/llm-azure.py +++ b/examples/update-settings/llm/llm-azure.py @@ -50,18 +50,18 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = AzureLLMService( - api_key=os.getenv("AZURE_CHATGPT_API_KEY"), - endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + api_key=os.environ["AZURE_CHATGPT_API_KEY"], + endpoint=os.environ["AZURE_CHATGPT_ENDPOINT"], settings=AzureLLMService.Settings( model=os.getenv("AZURE_CHATGPT_MODEL"), system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/update-settings/llm/llm-cerebras.py b/examples/update-settings/llm/llm-cerebras.py index edda71d49..232723a3e 100644 --- a/examples/update-settings/llm/llm-cerebras.py +++ b/examples/update-settings/llm/llm-cerebras.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = CerebrasLLMService( - api_key=os.getenv("CEREBRAS_API_KEY"), + api_key=os.environ["CEREBRAS_API_KEY"], settings=CerebrasLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-deepseek.py b/examples/update-settings/llm/llm-deepseek.py index adcb63259..a7df0469b 100644 --- a/examples/update-settings/llm/llm-deepseek.py +++ b/examples/update-settings/llm/llm-deepseek.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = DeepSeekLLMService( - api_key=os.getenv("DEEPSEEK_API_KEY"), + api_key=os.environ["DEEPSEEK_API_KEY"], settings=DeepSeekLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-fireworks.py b/examples/update-settings/llm/llm-fireworks.py index 3468d8dec..154bb63ce 100644 --- a/examples/update-settings/llm/llm-fireworks.py +++ b/examples/update-settings/llm/llm-fireworks.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = FireworksLLMService( - api_key=os.getenv("FIREWORKS_API_KEY"), + api_key=os.environ["FIREWORKS_API_KEY"], settings=FireworksLLMService.Settings( model="accounts/fireworks/models/gpt-oss-20b", system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/update-settings/llm/llm-gemini-live-vertex.py b/examples/update-settings/llm/llm-gemini-live-vertex.py index c18fb34c5..3992be309 100644 --- a/examples/update-settings/llm/llm-gemini-live-vertex.py +++ b/examples/update-settings/llm/llm-gemini-live-vertex.py @@ -49,9 +49,9 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") llm = GeminiLiveVertexLLMService( - credentials=os.getenv("GOOGLE_VERTEX_TEST_CREDENTIALS"), - project_id=os.getenv("GOOGLE_CLOUD_PROJECT_ID"), - location=os.getenv("GOOGLE_CLOUD_LOCATION"), + credentials=os.environ["GOOGLE_VERTEX_TEST_CREDENTIALS"], + project_id=os.environ["GOOGLE_CLOUD_PROJECT_ID"], + location=os.environ["GOOGLE_CLOUD_LOCATION"], settings=GeminiLiveVertexLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-gemini-live.py b/examples/update-settings/llm/llm-gemini-live.py index dafc9efe4..0c066648e 100644 --- a/examples/update-settings/llm/llm-gemini-live.py +++ b/examples/update-settings/llm/llm-gemini-live.py @@ -49,7 +49,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") llm = GeminiLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GeminiLiveLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-google-vertex.py b/examples/update-settings/llm/llm-google-vertex.py index b63b176bc..7c4ca8446 100644 --- a/examples/update-settings/llm/llm-google-vertex.py +++ b/examples/update-settings/llm/llm-google-vertex.py @@ -50,10 +50,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -61,8 +61,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = GoogleVertexLLMService( credentials=os.getenv("GOOGLE_VERTEX_TEST_CREDENTIALS"), - project_id=os.getenv("GOOGLE_CLOUD_PROJECT_ID"), - location=os.getenv("GOOGLE_CLOUD_LOCATION"), + project_id=os.environ["GOOGLE_CLOUD_PROJECT_ID"], + location=os.environ["GOOGLE_CLOUD_LOCATION"], settings=GoogleVertexLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-google.py b/examples/update-settings/llm/llm-google.py index 09a9d686d..69b474190 100644 --- a/examples/update-settings/llm/llm-google.py +++ b/examples/update-settings/llm/llm-google.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-grok-realtime.py b/examples/update-settings/llm/llm-grok-realtime.py index 304ccce8e..4ab28cdda 100644 --- a/examples/update-settings/llm/llm-grok-realtime.py +++ b/examples/update-settings/llm/llm-grok-realtime.py @@ -10,6 +10,7 @@ import os from dotenv import load_dotenv from loguru import logger +from pipecat.adapters.base_llm_adapter import LLMContextMessage from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame from pipecat.pipeline.pipeline import Pipeline @@ -50,9 +51,9 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - llm = GrokRealtimeLLMService(api_key=os.getenv("XAI_API_KEY")) + llm = GrokRealtimeLLMService(api_key=os.environ["XAI_API_KEY"]) - messages = [ + messages: list[LLMContextMessage] = [ { "role": "system", "content": "You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/update-settings/llm/llm-grok.py b/examples/update-settings/llm/llm-grok.py index dedb7b950..463b27807 100644 --- a/examples/update-settings/llm/llm-grok.py +++ b/examples/update-settings/llm/llm-grok.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = GrokLLMService( - api_key=os.getenv("XAI_API_KEY"), + api_key=os.environ["XAI_API_KEY"], settings=GrokLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-groq.py b/examples/update-settings/llm/llm-groq.py index 771a5cbe9..365d8b4d4 100644 --- a/examples/update-settings/llm/llm-groq.py +++ b/examples/update-settings/llm/llm-groq.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = GroqLLMService( - api_key=os.getenv("GROQ_API_KEY"), + api_key=os.environ["GROQ_API_KEY"], settings=GroqLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-mistral.py b/examples/update-settings/llm/llm-mistral.py index 5da4d47b7..4aa66197e 100644 --- a/examples/update-settings/llm/llm-mistral.py +++ b/examples/update-settings/llm/llm-mistral.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = MistralLLMService( - api_key=os.getenv("MISTRAL_API_KEY"), + api_key=os.environ["MISTRAL_API_KEY"], settings=MistralLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-nvidia.py b/examples/update-settings/llm/llm-nvidia.py index 06c45dd5d..a44083821 100644 --- a/examples/update-settings/llm/llm-nvidia.py +++ b/examples/update-settings/llm/llm-nvidia.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = NvidiaLLMService( - api_key=os.getenv("NVIDIA_API_KEY"), + api_key=os.environ["NVIDIA_API_KEY"], settings=NvidiaLLMService.Settings( model="meta/llama-3.1-405b-instruct", system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/update-settings/llm/llm-ollama.py b/examples/update-settings/llm/llm-ollama.py index 90ddcbfb0..ac9892bee 100644 --- a/examples/update-settings/llm/llm-ollama.py +++ b/examples/update-settings/llm/llm-ollama.py @@ -50,10 +50,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), diff --git a/examples/update-settings/llm/llm-openai-realtime.py b/examples/update-settings/llm/llm-openai-realtime.py index 83cdb9fa7..120d8a86e 100644 --- a/examples/update-settings/llm/llm-openai-realtime.py +++ b/examples/update-settings/llm/llm-openai-realtime.py @@ -10,6 +10,7 @@ import os from dotenv import load_dotenv from loguru import logger +from pipecat.adapters.base_llm_adapter import LLMContextMessage from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame from pipecat.pipeline.pipeline import Pipeline @@ -50,9 +51,9 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - llm = OpenAIRealtimeLLMService(api_key=os.getenv("OPENAI_API_KEY")) + llm = OpenAIRealtimeLLMService(api_key=os.environ["OPENAI_API_KEY"]) - messages = [ + messages: list[LLMContextMessage] = [ { "role": "system", "content": "You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/update-settings/llm/llm-openai-responses-http.py b/examples/update-settings/llm/llm-openai-responses-http.py index 051a2f71e..0e7d6ec44 100644 --- a/examples/update-settings/llm/llm-openai-responses-http.py +++ b/examples/update-settings/llm/llm-openai-responses-http.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesHttpLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIResponsesHttpLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-openai-responses.py b/examples/update-settings/llm/llm-openai-responses.py index 8c214b639..d4857bab1 100644 --- a/examples/update-settings/llm/llm-openai-responses.py +++ b/examples/update-settings/llm/llm-openai-responses.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIResponsesLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-openai.py b/examples/update-settings/llm/llm-openai.py index 15ee4554a..dee6ba8cf 100644 --- a/examples/update-settings/llm/llm-openai.py +++ b/examples/update-settings/llm/llm-openai.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-openrouter.py b/examples/update-settings/llm/llm-openrouter.py index 7b8aef6d8..3c554b921 100644 --- a/examples/update-settings/llm/llm-openrouter.py +++ b/examples/update-settings/llm/llm-openrouter.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenRouterLLMService( - api_key=os.getenv("OPENROUTER_API_KEY"), + api_key=os.environ["OPENROUTER_API_KEY"], settings=OpenRouterLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-perplexity.py b/examples/update-settings/llm/llm-perplexity.py index 99a40c614..d1e2aae4a 100644 --- a/examples/update-settings/llm/llm-perplexity.py +++ b/examples/update-settings/llm/llm-perplexity.py @@ -50,25 +50,18 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) - llm = PerplexityLLMService(api_key=os.getenv("PERPLEXITY_API_KEY")) + llm = PerplexityLLMService(api_key=os.environ["PERPLEXITY_API_KEY"]) - messages = [ - { - "role": "developer", - "content": "You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. Start by introducing yourself.", - }, - ] - - context = LLMContext(messages) + context = LLMContext() user_aggregator, assistant_aggregator = LLMContextAggregatorPair( context, user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), @@ -98,6 +91,9 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info(f"Client connected") + context.add_message( + {"role": "developer", "content": "Please introduce yourself to the user."} + ) await task.queue_frames([LLMRunFrame()]) await asyncio.sleep(10) diff --git a/examples/update-settings/llm/llm-qwen.py b/examples/update-settings/llm/llm-qwen.py index ff7d126d3..585a7c2da 100644 --- a/examples/update-settings/llm/llm-qwen.py +++ b/examples/update-settings/llm/llm-qwen.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = QwenLLMService( - api_key=os.getenv("QWEN_API_KEY"), + api_key=os.environ["QWEN_API_KEY"], settings=QwenLLMService.Settings( model="qwen2.5-72b-instruct", system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/update-settings/llm/llm-sambanova.py b/examples/update-settings/llm/llm-sambanova.py index 4f03db5f2..c543fcee2 100644 --- a/examples/update-settings/llm/llm-sambanova.py +++ b/examples/update-settings/llm/llm-sambanova.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = SambaNovaLLMService( - api_key=os.getenv("SAMBANOVA_API_KEY"), + api_key=os.environ["SAMBANOVA_API_KEY"], settings=SambaNovaLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-sarvam.py b/examples/update-settings/llm/llm-sarvam.py index 62102238f..2966f46b8 100644 --- a/examples/update-settings/llm/llm-sarvam.py +++ b/examples/update-settings/llm/llm-sarvam.py @@ -47,31 +47,26 @@ transport_params = { } -def _require_env(name: str) -> str: - value = os.getenv(name) - if not value: - raise ValueError(f"Environment variable `{name}` is required.") - return value - - async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info("Starting bot") stt = SarvamSTTService( settings=SarvamSTTService.Settings(model="saaras:v3"), - api_key=_require_env("SARVAM_API_KEY"), + api_key=os.environ["SARVAM_API_KEY"], ) tts = SarvamTTSService( settings=SarvamTTSService.Settings(model="bulbul:v3"), - api_key=_require_env("SARVAM_API_KEY"), + api_key=os.environ["SARVAM_API_KEY"], ) llm = SarvamLLMService( - api_key=_require_env("SARVAM_API_KEY"), - settings=SarvamLLMService.Settings(model="sarvam-30b"), - system_instruction=( - "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way." + api_key=os.environ["SARVAM_API_KEY"], + settings=SarvamLLMService.Settings( + model="sarvam-30b", + system_instruction=( + "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way." + ), ), ) diff --git a/examples/update-settings/llm/llm-together.py b/examples/update-settings/llm/llm-together.py index 3144f1906..2e1a8cb7a 100644 --- a/examples/update-settings/llm/llm-together.py +++ b/examples/update-settings/llm/llm-together.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = TogetherLLMService( - api_key=os.getenv("TOGETHER_API_KEY"), + api_key=os.environ["TOGETHER_API_KEY"], settings=TogetherLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/llm/llm-ultravox-realtime.py b/examples/update-settings/llm/llm-ultravox-realtime.py index f77aa4252..4c530735e 100644 --- a/examples/update-settings/llm/llm-ultravox-realtime.py +++ b/examples/update-settings/llm/llm-ultravox-realtime.py @@ -11,6 +11,7 @@ import os from dotenv import load_dotenv from loguru import logger +from pipecat.adapters.base_llm_adapter import LLMContextMessage from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMRunFrame, LLMUpdateSettingsFrame @@ -55,7 +56,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = UltravoxRealtimeLLMService( params=OneShotInputParams( - api_key=os.getenv("ULTRAVOX_API_KEY"), + api_key=os.environ["ULTRAVOX_API_KEY"], system_prompt=system_prompt, temperature=0.3, max_duration=datetime.timedelta(minutes=3), @@ -63,7 +64,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): one_shot_selected_tools=ToolsSchema(standard_tools=[]), ) - messages = [ + messages: list[LLMContextMessage] = [ { "role": "system", "content": system_prompt, diff --git a/examples/update-settings/stt/stt-assemblyai.py b/examples/update-settings/stt/stt-assemblyai.py index 8017b6021..276499bb1 100644 --- a/examples/update-settings/stt/stt-assemblyai.py +++ b/examples/update-settings/stt/stt-assemblyai.py @@ -51,21 +51,21 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = AssemblyAISTTService( - api_key=os.getenv("ASSEMBLYAI_API_KEY"), + api_key=os.environ["ASSEMBLYAI_API_KEY"], settings=AssemblyAISTTService.Settings( model="u3-rt-pro", ), ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Try saying difficult names like 'Xiomara', 'Saoirse', or 'Krzystof' to test transcription accuracy.", ), diff --git a/examples/update-settings/stt/stt-aws-transcribe.py b/examples/update-settings/stt/stt-aws-transcribe.py index 25c2f272a..503ecae39 100644 --- a/examples/update-settings/stt/stt-aws-transcribe.py +++ b/examples/update-settings/stt/stt-aws-transcribe.py @@ -54,14 +54,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): stt = AWSTranscribeSTTService() tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-azure.py b/examples/update-settings/stt/stt-azure.py index adc242d3c..e1f8b62ad 100644 --- a/examples/update-settings/stt/stt-azure.py +++ b/examples/update-settings/stt/stt-azure.py @@ -52,19 +52,19 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = AzureSTTService( - api_key=os.getenv("AZURE_SPEECH_API_KEY"), + api_key=os.environ["AZURE_SPEECH_API_KEY"], region=os.getenv("AZURE_SPEECH_REGION"), ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-cartesia.py b/examples/update-settings/stt/stt-cartesia.py index 79daec9fd..2f5602b8d 100644 --- a/examples/update-settings/stt/stt-cartesia.py +++ b/examples/update-settings/stt/stt-cartesia.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = CartesiaSTTService(api_key=os.getenv("CARTESIA_API_KEY")) + stt = CartesiaSTTService(api_key=os.environ["CARTESIA_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-deepgram-flux.py b/examples/update-settings/stt/stt-deepgram-flux.py index 24bb8b7ad..f8810e2fd 100644 --- a/examples/update-settings/stt/stt-deepgram-flux.py +++ b/examples/update-settings/stt/stt-deepgram-flux.py @@ -29,6 +29,7 @@ from pipecat.transcriptions.language import Language from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams +from pipecat.turns.user_turn_strategies import ExternalUserTurnStrategies load_dotenv(override=True) @@ -51,17 +52,26 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramFluxSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + # Start with the multilingual model and broad hints so Flux can auto-detect + # across English and Spanish. TranscriptionFrame.language will reflect + # whichever language Flux detected for each turn. + stt = DeepgramFluxSTTService( + api_key=os.environ["DEEPGRAM_API_KEY"], + settings=DeepgramFluxSTTService.Settings( + model="flux-general-multi", + language_hints=[Language.EN, Language.ES], + ), + ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), @@ -70,7 +80,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): context = LLMContext() user_aggregator, assistant_aggregator = LLMContextAggregatorPair( context, - user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + user_params=LLMUserAggregatorParams( + user_turn_strategies=ExternalUserTurnStrategies(), + vad_analyzer=SileroVADAnalyzer(), + ), ) pipeline = Pipeline( @@ -108,17 +121,21 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info("Updating Deepgram Flux STT settings: eot_threshold, keyterm") await task.queue_frame( STTUpdateSettingsFrame( - delta=DeepgramFluxSTTSettings( + delta=DeepgramFluxSTTService.Settings( eot_threshold=0.8, keyterm=["Pipecat", "Deepgram"], ) ) ) + # Detect-then-lock: narrow the hints to a single language mid-stream. + # Sent as a Configure message — no reconnect needed. await asyncio.sleep(10) - logger.info("Updating Deepgram Flux STT settings: language=es") + logger.info("Updating Deepgram Flux STT settings: language_hints=[es]") await task.queue_frame( - STTUpdateSettingsFrame(delta=DeepgramFluxSTTService.Settings(language=Language.ES)) + STTUpdateSettingsFrame( + delta=DeepgramFluxSTTService.Settings(language_hints=[Language.ES]) + ) ) @transport.event_handler("on_client_disconnected") diff --git a/examples/update-settings/stt/stt-deepgram-sagemaker.py b/examples/update-settings/stt/stt-deepgram-sagemaker.py index aa61eaa1e..05a655abc 100644 --- a/examples/update-settings/stt/stt-deepgram-sagemaker.py +++ b/examples/update-settings/stt/stt-deepgram-sagemaker.py @@ -52,19 +52,19 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = DeepgramSageMakerSTTService( - endpoint_name=os.getenv("SAGEMAKER_STT_ENDPOINT_NAME"), - region=os.getenv("AWS_REGION"), + endpoint_name=os.environ["SAGEMAKER_STT_ENDPOINT_NAME"], + region=os.environ["AWS_REGION"], ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-deepgram.py b/examples/update-settings/stt/stt-deepgram.py index a083244d0..9f345b1c3 100644 --- a/examples/update-settings/stt/stt-deepgram.py +++ b/examples/update-settings/stt/stt-deepgram.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-elevenlabs-realtime.py b/examples/update-settings/stt/stt-elevenlabs-realtime.py index f2cba36ec..8a195ad48 100644 --- a/examples/update-settings/stt/stt-elevenlabs-realtime.py +++ b/examples/update-settings/stt/stt-elevenlabs-realtime.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = ElevenLabsRealtimeSTTService(api_key=os.getenv("ELEVENLABS_API_KEY")) + stt = ElevenLabsRealtimeSTTService(api_key=os.environ["ELEVENLABS_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-elevenlabs.py b/examples/update-settings/stt/stt-elevenlabs.py index 03efdc71e..3b3623264 100644 --- a/examples/update-settings/stt/stt-elevenlabs.py +++ b/examples/update-settings/stt/stt-elevenlabs.py @@ -54,19 +54,19 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): async with aiohttp.ClientSession() as session: stt = ElevenLabsSTTService( - api_key=os.getenv("ELEVENLABS_API_KEY"), + api_key=os.environ["ELEVENLABS_API_KEY"], aiohttp_session=session, ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-fal.py b/examples/update-settings/stt/stt-fal.py index 83b7dbc79..5dccd4d36 100644 --- a/examples/update-settings/stt/stt-fal.py +++ b/examples/update-settings/stt/stt-fal.py @@ -53,14 +53,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): stt = FalSTTService(api_key=os.getenv("FAL_KEY")) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-gladia.py b/examples/update-settings/stt/stt-gladia.py index 972456853..4449eff06 100644 --- a/examples/update-settings/stt/stt-gladia.py +++ b/examples/update-settings/stt/stt-gladia.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = GladiaSTTService(api_key=os.getenv("GLADIA_API_KEY")) + stt = GladiaSTTService(api_key=os.environ["GLADIA_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-google.py b/examples/update-settings/stt/stt-google.py index 58b0ff357..76ea1325d 100644 --- a/examples/update-settings/stt/stt-google.py +++ b/examples/update-settings/stt/stt-google.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = GoogleSTTService(credentials=os.getenv("GOOGLE_TEST_CREDENTIALS")) + stt = GoogleSTTService(credentials=os.environ["GOOGLE_TEST_CREDENTIALS"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-gradium.py b/examples/update-settings/stt/stt-gradium.py index aebced030..876b20260 100644 --- a/examples/update-settings/stt/stt-gradium.py +++ b/examples/update-settings/stt/stt-gradium.py @@ -51,19 +51,19 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = GradiumSTTService( - api_key=os.getenv("GRADIUM_API_KEY"), + api_key=os.environ["GRADIUM_API_KEY"], api_endpoint_base_url="wss://us.api.gradium.ai/api/speech/asr", ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-groq.py b/examples/update-settings/stt/stt-groq.py index f612ba05b..033ed4c03 100644 --- a/examples/update-settings/stt/stt-groq.py +++ b/examples/update-settings/stt/stt-groq.py @@ -51,18 +51,18 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = GroqSTTService( - api_key=os.getenv("GROQ_API_KEY"), + api_key=os.environ["GROQ_API_KEY"], ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-nvidia-segmented.py b/examples/update-settings/stt/stt-nvidia-segmented.py index d2880d446..1cce6fba2 100644 --- a/examples/update-settings/stt/stt-nvidia-segmented.py +++ b/examples/update-settings/stt/stt-nvidia-segmented.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = NvidiaSegmentedSTTService(api_key=os.getenv("NVIDIA_API_KEY")) + stt = NvidiaSegmentedSTTService(api_key=os.environ["NVIDIA_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-nvidia.py b/examples/update-settings/stt/stt-nvidia.py index e339218e8..499269f86 100644 --- a/examples/update-settings/stt/stt-nvidia.py +++ b/examples/update-settings/stt/stt-nvidia.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = NvidiaSTTService(api_key=os.getenv("NVIDIA_API_KEY")) + stt = NvidiaSTTService(api_key=os.environ["NVIDIA_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-openai-realtime.py b/examples/update-settings/stt/stt-openai-realtime.py index e5366eb7f..33c4c4e7d 100644 --- a/examples/update-settings/stt/stt-openai-realtime.py +++ b/examples/update-settings/stt/stt-openai-realtime.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = OpenAIRealtimeSTTService(api_key=os.getenv("OPENAI_API_KEY")) + stt = OpenAIRealtimeSTTService(api_key=os.environ["OPENAI_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-sarvam.py b/examples/update-settings/stt/stt-sarvam.py index ddb76275d..60da61359 100644 --- a/examples/update-settings/stt/stt-sarvam.py +++ b/examples/update-settings/stt/stt-sarvam.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = SarvamSTTService(api_key=os.getenv("SARVAM_API_KEY")) + stt = SarvamSTTService(api_key=os.environ["SARVAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-soniox.py b/examples/update-settings/stt/stt-soniox.py index 05b3c267f..adf3fb547 100644 --- a/examples/update-settings/stt/stt-soniox.py +++ b/examples/update-settings/stt/stt-soniox.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = SonioxSTTService(api_key=os.getenv("SONIOX_API_KEY")) + stt = SonioxSTTService(api_key=os.environ["SONIOX_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-speechmatics.py b/examples/update-settings/stt/stt-speechmatics.py index 93c97c827..f92d2ff7e 100644 --- a/examples/update-settings/stt/stt-speechmatics.py +++ b/examples/update-settings/stt/stt-speechmatics.py @@ -52,7 +52,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = SpeechmaticsSTTService( - api_key=os.getenv("SPEECHMATICS_API_KEY"), + api_key=os.environ["SPEECHMATICS_API_KEY"], settings=SpeechmaticsSTTService.Settings( enable_diarization=True, speaker_active_format="<{speaker_id}>{text}", @@ -61,14 +61,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-whisper-api.py b/examples/update-settings/stt/stt-whisper-api.py index 84dea00a2..372e6a36e 100644 --- a/examples/update-settings/stt/stt-whisper-api.py +++ b/examples/update-settings/stt/stt-whisper-api.py @@ -56,18 +56,18 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # - SambaNova # - Groq stt = OpenAISTTService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-whisper-mlx.py b/examples/update-settings/stt/stt-whisper-mlx.py index a13c053cf..7e9e5e4e1 100644 --- a/examples/update-settings/stt/stt-whisper-mlx.py +++ b/examples/update-settings/stt/stt-whisper-mlx.py @@ -57,14 +57,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/stt/stt-whisper.py b/examples/update-settings/stt/stt-whisper.py index d09c4f3b4..e25f3915b 100644 --- a/examples/update-settings/stt/stt-whisper.py +++ b/examples/update-settings/stt/stt-whisper.py @@ -57,14 +57,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-asyncai-http.py b/examples/update-settings/tts/tts-asyncai-http.py index 93ca099b0..5f6342114 100644 --- a/examples/update-settings/tts/tts-asyncai-http.py +++ b/examples/update-settings/tts/tts-asyncai-http.py @@ -55,7 +55,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = AsyncAIHttpTTSService( api_key=os.getenv("ASYNCAI_API_KEY", ""), @@ -66,7 +66,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-asyncai.py b/examples/update-settings/tts/tts-asyncai.py index a24e37c9e..3d7701288 100644 --- a/examples/update-settings/tts/tts-asyncai.py +++ b/examples/update-settings/tts/tts-asyncai.py @@ -51,7 +51,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = AsyncAITTSService( api_key=os.getenv("ASYNCAI_API_KEY", ""), @@ -61,7 +61,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-aws-polly.py b/examples/update-settings/tts/tts-aws-polly.py index b00c6982b..6ddeaf04b 100644 --- a/examples/update-settings/tts/tts-aws-polly.py +++ b/examples/update-settings/tts/tts-aws-polly.py @@ -50,12 +50,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = AWSPollyTTSService() llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-azure-http.py b/examples/update-settings/tts/tts-azure-http.py index a0193b063..5b3d6adb3 100644 --- a/examples/update-settings/tts/tts-azure-http.py +++ b/examples/update-settings/tts/tts-azure-http.py @@ -50,15 +50,15 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = AzureHttpTTSService( - api_key=os.getenv("AZURE_SPEECH_API_KEY"), - region=os.getenv("AZURE_SPEECH_REGION"), + api_key=os.environ["AZURE_SPEECH_API_KEY"], + region=os.environ["AZURE_SPEECH_REGION"], ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-azure.py b/examples/update-settings/tts/tts-azure.py index 46daa6781..a3a1eb822 100644 --- a/examples/update-settings/tts/tts-azure.py +++ b/examples/update-settings/tts/tts-azure.py @@ -50,15 +50,15 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = AzureTTSService( - api_key=os.getenv("AZURE_SPEECH_API_KEY"), - region=os.getenv("AZURE_SPEECH_REGION"), + api_key=os.environ["AZURE_SPEECH_API_KEY"], + region=os.environ["AZURE_SPEECH_REGION"], ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-camb.py b/examples/update-settings/tts/tts-camb.py index 97166ad0f..874489d97 100644 --- a/examples/update-settings/tts/tts-camb.py +++ b/examples/update-settings/tts/tts-camb.py @@ -51,12 +51,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) - tts = CambTTSService(api_key=os.getenv("CAMB_API_KEY")) + tts = CambTTSService(api_key=os.environ["CAMB_API_KEY"]) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-cartesia-http.py b/examples/update-settings/tts/tts-cartesia-http.py index 84d3188af..b126d1dee 100644 --- a/examples/update-settings/tts/tts-cartesia-http.py +++ b/examples/update-settings/tts/tts-cartesia-http.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaHttpTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaHttpTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-cartesia.py b/examples/update-settings/tts/tts-cartesia.py index 71d804875..7f267a4d7 100644 --- a/examples/update-settings/tts/tts-cartesia.py +++ b/examples/update-settings/tts/tts-cartesia.py @@ -52,17 +52,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-deepgram-http.py b/examples/update-settings/tts/tts-deepgram-http.py index 16083e95e..0aa24c4e5 100644 --- a/examples/update-settings/tts/tts-deepgram-http.py +++ b/examples/update-settings/tts/tts-deepgram-http.py @@ -54,15 +54,15 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = DeepgramHttpTTSService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], aiohttp_session=session, ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-deepgram-sagemaker.py b/examples/update-settings/tts/tts-deepgram-sagemaker.py index 1dfd2a365..47329b1f3 100644 --- a/examples/update-settings/tts/tts-deepgram-sagemaker.py +++ b/examples/update-settings/tts/tts-deepgram-sagemaker.py @@ -50,16 +50,16 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = DeepgramSageMakerTTSService( - endpoint_name=os.getenv("SAGEMAKER_TTS_ENDPOINT_NAME"), - region=os.getenv("AWS_REGION"), + endpoint_name=os.environ["SAGEMAKER_TTS_ENDPOINT_NAME"], + region=os.environ["AWS_REGION"], voice="aura-2-helena-en", ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-deepgram.py b/examples/update-settings/tts/tts-deepgram.py index 596d12dfa..8e532937a 100644 --- a/examples/update-settings/tts/tts-deepgram.py +++ b/examples/update-settings/tts/tts-deepgram.py @@ -50,12 +50,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) - tts = DeepgramTTSService(api_key=os.getenv("DEEPGRAM_API_KEY")) + tts = DeepgramTTSService(api_key=os.environ["DEEPGRAM_API_KEY"]) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-elevenlabs-http.py b/examples/update-settings/tts/tts-elevenlabs-http.py index 60dab3017..ac85a1452 100644 --- a/examples/update-settings/tts/tts-elevenlabs-http.py +++ b/examples/update-settings/tts/tts-elevenlabs-http.py @@ -54,16 +54,21 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = ElevenLabsHttpTTSService( - api_key=os.getenv("ELEVENLABS_API_KEY"), - settings=ElevenLabsHttpTTSService.Settings(voice=os.getenv("ELEVENLABS_VOICE_ID")), + api_key=os.environ["ELEVENLABS_API_KEY"], + settings=ElevenLabsHttpTTSService.Settings( + voice=os.getenv( + "ELEVENLABS_VOICE_ID", + "Xb7hH8MSUJpSbSDYk0k2", + ) + ), aiohttp_session=session, ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-elevenlabs.py b/examples/update-settings/tts/tts-elevenlabs.py index 83de8165f..9aeadf993 100644 --- a/examples/update-settings/tts/tts-elevenlabs.py +++ b/examples/update-settings/tts/tts-elevenlabs.py @@ -50,15 +50,20 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = ElevenLabsTTSService( - api_key=os.getenv("ELEVENLABS_API_KEY"), - settings=ElevenLabsTTSService.Settings(voice=os.getenv("ELEVENLABS_VOICE_ID")), + api_key=os.environ["ELEVENLABS_API_KEY"], + settings=ElevenLabsTTSService.Settings( + voice=os.getenv( + "ELEVENLABS_VOICE_ID", + "Xb7hH8MSUJpSbSDYk0k2", + ) + ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), @@ -109,7 +114,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info("Updating ElevenLabs TTS settings: switching to a different voice") await task.queue_frame( TTSUpdateSettingsFrame( - delta=ElevenLabsTTSService.Settings(voice=os.getenv("ELEVENLABS_VOICE_ID_ALT")) + delta=ElevenLabsTTSService.Settings( + voice=os.getenv( + "ELEVENLABS_VOICE_ID_ALT", + "CwhRBWXzGAHq8TQ4Fs17", + ) + ) ) ) diff --git a/examples/update-settings/tts/tts-fish.py b/examples/update-settings/tts/tts-fish.py index f486ad9d1..44566d915 100644 --- a/examples/update-settings/tts/tts-fish.py +++ b/examples/update-settings/tts/tts-fish.py @@ -50,17 +50,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = FishAudioTTSService( - api_key=os.getenv("FISH_API_KEY"), + api_key=os.environ["FISH_API_KEY"], settings=FishAudioTTSService.Settings( voice="4ce7e917cedd4bc2bb2e6ff3a46acaa1" ), # Barack Obama ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-gemini.py b/examples/update-settings/tts/tts-gemini.py index e984c3575..a6b56ac54 100644 --- a/examples/update-settings/tts/tts-gemini.py +++ b/examples/update-settings/tts/tts-gemini.py @@ -51,10 +51,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = GeminiTTSService( - credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + credentials=os.environ["GOOGLE_TEST_CREDENTIALS"], settings=GeminiTTSService.Settings( model="gemini-2.5-flash-tts", voice="Charon", @@ -64,7 +64,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-google-http.py b/examples/update-settings/tts/tts-google-http.py index b9a605560..eef39d8f8 100644 --- a/examples/update-settings/tts/tts-google-http.py +++ b/examples/update-settings/tts/tts-google-http.py @@ -50,12 +50,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) - tts = GoogleHttpTTSService(credentials=os.getenv("GOOGLE_TEST_CREDENTIALS")) + tts = GoogleHttpTTSService(credentials=os.environ["GOOGLE_TEST_CREDENTIALS"]) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-google-stream.py b/examples/update-settings/tts/tts-google-stream.py index a1c5129a6..2614022e7 100644 --- a/examples/update-settings/tts/tts-google-stream.py +++ b/examples/update-settings/tts/tts-google-stream.py @@ -50,12 +50,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) - tts = GoogleTTSService(credentials=os.getenv("GOOGLE_TEST_CREDENTIALS")) + tts = GoogleTTSService(credentials=os.environ["GOOGLE_TEST_CREDENTIALS"]) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-gradium.py b/examples/update-settings/tts/tts-gradium.py index 7498557a9..3750b004a 100644 --- a/examples/update-settings/tts/tts-gradium.py +++ b/examples/update-settings/tts/tts-gradium.py @@ -50,16 +50,16 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = GradiumTTSService( - api_key=os.getenv("GRADIUM_API_KEY"), + api_key=os.environ["GRADIUM_API_KEY"], settings=GradiumTTSService.Settings(voice="YTpq7expH9539ERJ"), url="wss://us.api.gradium.ai/api/speech/tts", ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-groq.py b/examples/update-settings/tts/tts-groq.py index 757276c6a..1926dfa96 100644 --- a/examples/update-settings/tts/tts-groq.py +++ b/examples/update-settings/tts/tts-groq.py @@ -50,12 +50,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) - tts = GroqTTSService(api_key=os.getenv("GROQ_API_KEY")) + tts = GroqTTSService(api_key=os.environ["GROQ_API_KEY"]) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-hume.py b/examples/update-settings/tts/tts-hume.py index 034ff6797..6399a908a 100644 --- a/examples/update-settings/tts/tts-hume.py +++ b/examples/update-settings/tts/tts-hume.py @@ -50,7 +50,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = HumeTTSService( api_key=os.getenv("HUME_API_KEY"), @@ -58,7 +58,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-inworld-http.py b/examples/update-settings/tts/tts-inworld-http.py index 41b960dd4..28ea6c81d 100644 --- a/examples/update-settings/tts/tts-inworld-http.py +++ b/examples/update-settings/tts/tts-inworld-http.py @@ -54,12 +54,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) - tts = InworldHttpTTSService(api_key=os.getenv("INWORLD_API_KEY"), aiohttp_session=session) + tts = InworldHttpTTSService(api_key=os.environ["INWORLD_API_KEY"], aiohttp_session=session) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-inworld.py b/examples/update-settings/tts/tts-inworld.py index 9f40247dd..9bdd706e5 100644 --- a/examples/update-settings/tts/tts-inworld.py +++ b/examples/update-settings/tts/tts-inworld.py @@ -50,12 +50,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) - tts = InworldTTSService(api_key=os.getenv("INWORLD_API_KEY")) + tts = InworldTTSService(api_key=os.environ["INWORLD_API_KEY"]) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-kokoro.py b/examples/update-settings/tts/tts-kokoro.py index 8c7e1f492..053dd630b 100644 --- a/examples/update-settings/tts/tts-kokoro.py +++ b/examples/update-settings/tts/tts-kokoro.py @@ -50,7 +50,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = KokoroTTSService( settings=KokoroTTSService.Settings( @@ -59,7 +59,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-lmnt.py b/examples/update-settings/tts/tts-lmnt.py index d700e22c2..68f5f6d12 100644 --- a/examples/update-settings/tts/tts-lmnt.py +++ b/examples/update-settings/tts/tts-lmnt.py @@ -50,15 +50,15 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = LmntTTSService( - api_key=os.getenv("LMNT_API_KEY"), + api_key=os.environ["LMNT_API_KEY"], settings=LmntTTSService.Settings(voice="lily"), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-minimax.py b/examples/update-settings/tts/tts-minimax.py index b77e84554..3a02adce0 100644 --- a/examples/update-settings/tts/tts-minimax.py +++ b/examples/update-settings/tts/tts-minimax.py @@ -52,16 +52,16 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = MiniMaxHttpTTSService( - api_key=os.getenv("MINIMAX_API_KEY", ""), - group_id=os.getenv("MINIMAX_GROUP_ID", ""), + api_key=os.environ["MINIMAX_API_KEY"], + group_id=os.environ["MINIMAX_GROUP_ID"], aiohttp_session=session, ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-neuphonic-http.py b/examples/update-settings/tts/tts-neuphonic-http.py index 3db9e5d2e..a55d6fd6c 100644 --- a/examples/update-settings/tts/tts-neuphonic-http.py +++ b/examples/update-settings/tts/tts-neuphonic-http.py @@ -51,16 +51,16 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) async with aiohttp.ClientSession() as session: tts = NeuphonicHttpTTSService( - api_key=os.getenv("NEUPHONIC_API_KEY"), + api_key=os.environ["NEUPHONIC_API_KEY"], aiohttp_session=session, ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-neuphonic.py b/examples/update-settings/tts/tts-neuphonic.py index bb10f8699..fd37ec40e 100644 --- a/examples/update-settings/tts/tts-neuphonic.py +++ b/examples/update-settings/tts/tts-neuphonic.py @@ -50,12 +50,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) - tts = NeuphonicTTSService(api_key=os.getenv("NEUPHONIC_API_KEY")) + tts = NeuphonicTTSService(api_key=os.environ["NEUPHONIC_API_KEY"]) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-nvidia.py b/examples/update-settings/tts/tts-nvidia.py index 2b41e1cdd..e136db033 100644 --- a/examples/update-settings/tts/tts-nvidia.py +++ b/examples/update-settings/tts/tts-nvidia.py @@ -51,12 +51,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) - tts = NvidiaTTSService(api_key=os.getenv("NVIDIA_API_KEY")) + tts = NvidiaTTSService(api_key=os.environ["NVIDIA_API_KEY"]) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-openai.py b/examples/update-settings/tts/tts-openai.py index c75e56513..e23db32ac 100644 --- a/examples/update-settings/tts/tts-openai.py +++ b/examples/update-settings/tts/tts-openai.py @@ -50,12 +50,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) - tts = OpenAITTSService(api_key=os.getenv("OPENAI_API_KEY")) + tts = OpenAITTSService(api_key=os.environ["OPENAI_API_KEY"]) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-piper-http.py b/examples/update-settings/tts/tts-piper-http.py index 685f81dd5..ed943b5b0 100644 --- a/examples/update-settings/tts/tts-piper-http.py +++ b/examples/update-settings/tts/tts-piper-http.py @@ -53,16 +53,16 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = PiperHttpTTSService( - base_url=os.getenv("PIPER_BASE_URL"), + base_url=os.environ["PIPER_BASE_URL"], aiohttp_session=session, settings=PiperHttpTTSService.Settings(voice="en_US-ryan-high"), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-piper.py b/examples/update-settings/tts/tts-piper.py index df10d999a..710a9ad7f 100644 --- a/examples/update-settings/tts/tts-piper.py +++ b/examples/update-settings/tts/tts-piper.py @@ -50,7 +50,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = PiperTTSService( settings=PiperTTSService.Settings( @@ -59,7 +59,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-resembleai.py b/examples/update-settings/tts/tts-resembleai.py index ae295cfa2..3f3a5e271 100644 --- a/examples/update-settings/tts/tts-resembleai.py +++ b/examples/update-settings/tts/tts-resembleai.py @@ -50,15 +50,15 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = ResembleAITTSService( - api_key=os.getenv("RESEMBLE_API_KEY"), - settings=ResembleAITTSService.Settings(voice=os.getenv("RESEMBLE_VOICE_UUID")), + api_key=os.environ["RESEMBLE_API_KEY"], + settings=ResembleAITTSService.Settings(voice=os.environ["RESEMBLE_VOICE_UUID"]), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), @@ -103,7 +103,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info("Updating ResembleAI TTS settings: voice (changed)") await task.queue_frame( TTSUpdateSettingsFrame( - delta=ResembleAITTSService.Settings(voice=os.getenv("RESEMBLE_VOICE_UUID_ALT")) + delta=ResembleAITTSService.Settings(voice=os.environ["RESEMBLE_VOICE_UUID_ALT"]) ) ) diff --git a/examples/update-settings/tts/tts-rime-http.py b/examples/update-settings/tts/tts-rime-http.py index af69dcda5..3e5b2d2e7 100644 --- a/examples/update-settings/tts/tts-rime-http.py +++ b/examples/update-settings/tts/tts-rime-http.py @@ -54,16 +54,16 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = RimeHttpTTSService( - api_key=os.getenv("RIME_API_KEY"), + api_key=os.environ["RIME_API_KEY"], settings=RimeHttpTTSService.Settings(voice="eva"), aiohttp_session=session, ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-rime.py b/examples/update-settings/tts/tts-rime.py index 44bc69e13..314eeeb05 100644 --- a/examples/update-settings/tts/tts-rime.py +++ b/examples/update-settings/tts/tts-rime.py @@ -50,15 +50,15 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = RimeTTSService( - api_key=os.getenv("RIME_API_KEY"), + api_key=os.environ["RIME_API_KEY"], settings=RimeTTSService.Settings(voice="luna"), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-sarvam-http.py b/examples/update-settings/tts/tts-sarvam-http.py index e9a35f8cd..ec6911f5f 100644 --- a/examples/update-settings/tts/tts-sarvam-http.py +++ b/examples/update-settings/tts/tts-sarvam-http.py @@ -54,12 +54,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) - tts = SarvamHttpTTSService(api_key=os.getenv("SARVAM_API_KEY"), aiohttp_session=session) + tts = SarvamHttpTTSService(api_key=os.environ["SARVAM_API_KEY"], aiohttp_session=session) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-sarvam.py b/examples/update-settings/tts/tts-sarvam.py index 77bb80d57..9cb24ad93 100644 --- a/examples/update-settings/tts/tts-sarvam.py +++ b/examples/update-settings/tts/tts-sarvam.py @@ -50,12 +50,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) - tts = SarvamTTSService(api_key=os.getenv("SARVAM_API_KEY")) + tts = SarvamTTSService(api_key=os.environ["SARVAM_API_KEY"]) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-speechmatics.py b/examples/update-settings/tts/tts-speechmatics.py index 4cefa1cc8..a4515739c 100644 --- a/examples/update-settings/tts/tts-speechmatics.py +++ b/examples/update-settings/tts/tts-speechmatics.py @@ -51,16 +51,16 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) async with aiohttp.ClientSession() as session: tts = SpeechmaticsTTSService( - api_key=os.getenv("SPEECHMATICS_API_KEY"), + api_key=os.environ["SPEECHMATICS_API_KEY"], aiohttp_session=session, ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/update-settings/tts/tts-xtts.py b/examples/update-settings/tts/tts-xtts.py index b1fb4f309..0145e8c64 100644 --- a/examples/update-settings/tts/tts-xtts.py +++ b/examples/update-settings/tts/tts-xtts.py @@ -53,7 +53,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = XTTSService( aiohttp_session=session, @@ -64,7 +64,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/video-avatar/video-avatar-heygen-transport.py b/examples/video-avatar/video-avatar-heygen-transport.py index 47ec01abd..0ea504520 100644 --- a/examples/video-avatar/video-avatar-heygen-transport.py +++ b/examples/video-avatar/video-avatar-heygen-transport.py @@ -37,7 +37,7 @@ logger.add(sys.stderr, level="DEBUG") async def main(): async with aiohttp.ClientSession() as session: transport = HeyGenTransport( - api_key=os.getenv("HEYGEN_LIVE_AVATAR_API_KEY"), + api_key=os.environ["HEYGEN_LIVE_AVATAR_API_KEY"], service_type=ServiceType.LIVE_AVATAR, session=session, params=HeyGenParams( @@ -52,17 +52,17 @@ async def main(): ), ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="00967b2f-88a6-4a31-8153-110a92134b9f", ), ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( system_instruction="You are a helpful assistant. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Be succinct and respond to what the user said in a creative and helpful way.", ), diff --git a/examples/video-avatar/video-avatar-heygen-video-service.py b/examples/video-avatar/video-avatar-heygen-video-service.py index 35d2e3221..a48c42758 100644 --- a/examples/video-avatar/video-avatar-heygen-video-service.py +++ b/examples/video-avatar/video-avatar-heygen-video-service.py @@ -59,24 +59,24 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="00967b2f-88a6-4a31-8153-110a92134b9f", ), ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( system_instruction="You are a helpful assistant. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Be succinct and respond to what the user said in a creative and helpful way.", ), ) heyGen = HeyGenVideoService( - api_key=os.getenv("HEYGEN_LIVE_AVATAR_API_KEY"), + api_key=os.environ["HEYGEN_LIVE_AVATAR_API_KEY"], service_type=ServiceType.LIVE_AVATAR, session=session, session_request=LiveAvatarNewSessionRequest( diff --git a/examples/video-avatar/video-avatar-lemonslice-transport.py b/examples/video-avatar/video-avatar-lemonslice-transport.py index 52cd1ca25..08dd48aa3 100644 --- a/examples/video-avatar/video-avatar-lemonslice-transport.py +++ b/examples/video-avatar/video-avatar-lemonslice-transport.py @@ -41,7 +41,7 @@ async def main(): async with aiohttp.ClientSession() as session: transport = LemonSliceTransport( bot_name="Pipecat", - api_key=os.getenv("LEMONSLICE_API_KEY"), + api_key=os.environ["LEMONSLICE_API_KEY"], session=session, session_request=LemonSliceNewSessionRequest( agent_id=os.getenv("LEMONSLICE_AGENT_ID"), @@ -53,10 +53,10 @@ async def main(): ), ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) llm = GroqLLMService( - api_key=os.getenv("GROQ_API_KEY"), + api_key=os.environ["GROQ_API_KEY"], settings=GroqLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/video-avatar/video-avatar-simli-video-service.py b/examples/video-avatar/video-avatar-simli-video-service.py index 76c2dd99d..c91623253 100644 --- a/examples/video-avatar/video-avatar-simli-video-service.py +++ b/examples/video-avatar/video-avatar-simli-video-service.py @@ -56,22 +56,22 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", ), ) simli_ai = SimliVideoService( - api_key=os.getenv("SIMLI_API_KEY"), + api_key=os.environ["SIMLI_API_KEY"], face_id="cace3ef7-a4c4-425d-a8cf-a5358eb0c427", ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/video-avatar/video-avatar-tavus-transport.py b/examples/video-avatar/video-avatar-tavus-transport.py index db19e650e..566538dbb 100644 --- a/examples/video-avatar/video-avatar-tavus-transport.py +++ b/examples/video-avatar/video-avatar-tavus-transport.py @@ -37,8 +37,8 @@ async def main(): async with aiohttp.ClientSession() as session: transport = TavusTransport( bot_name="Pipecat bot", - api_key=os.getenv("TAVUS_API_KEY"), - replica_id=os.getenv("TAVUS_REPLICA_ID"), + api_key=os.environ["TAVUS_API_KEY"], + replica_id=os.environ["TAVUS_REPLICA_ID"], session=session, params=TavusParams( audio_in_enabled=True, @@ -47,17 +47,17 @@ async def main(): ), ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="a167e0f3-df7e-4d52-a9c3-f949145efdab", ), ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/video-avatar/video-avatar-tavus-video-service.py b/examples/video-avatar/video-avatar-tavus-video-service.py index 5658951b5..ce3075074 100644 --- a/examples/video-avatar/video-avatar-tavus-video-service.py +++ b/examples/video-avatar/video-avatar-tavus-video-service.py @@ -59,25 +59,25 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="a167e0f3-df7e-4d52-a9c3-f949145efdab", ), ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), ) tavus = TavusVideoService( - api_key=os.getenv("TAVUS_API_KEY"), - replica_id=os.getenv("TAVUS_REPLICA_ID"), + api_key=os.environ["TAVUS_API_KEY"], + replica_id=os.environ["TAVUS_REPLICA_ID"], session=session, ) diff --git a/examples/video-processing/video-processing.py b/examples/video-processing/video-processing.py index 531c7c940..812de5b03 100644 --- a/examples/video-processing/video-processing.py +++ b/examples/video-processing/video-processing.py @@ -96,20 +96,13 @@ Respond to what the user said in a creative and helpful way. Keep your responses async def run_bot(pipecat_transport): llm = GeminiLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck transcribe_user_audio=True, system_instruction=SYSTEM_INSTRUCTION, ) - messages = [ - { - "role": "developer", - "content": "Start by greeting the user warmly and introducing yourself.", - } - ] - - context = LLMContext(messages) + context = LLMContext() user_aggregator, assistant_aggregator = LLMContextAggregatorPair( context, user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), @@ -141,6 +134,12 @@ async def run_bot(pipecat_transport): async def on_client_ready(rtvi): logger.info("Pipecat client ready.") # Kick off the conversation. + context.add_message( + { + "role": "developer", + "content": "Start by greeting the user warmly and introducing yourself.", + } + ) await task.queue_frames([LLMRunFrame()]) @pipecat_transport.event_handler("on_client_connected") diff --git a/examples/vision/vision-anthropic.py b/examples/vision/vision-anthropic.py index 642885dcf..d02cf8b95 100644 --- a/examples/vision/vision-anthropic.py +++ b/examples/vision/vision-anthropic.py @@ -49,17 +49,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), + api_key=os.environ["ANTHROPIC_API_KEY"], settings=AnthropicLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You are also able to describe images.", ), diff --git a/examples/vision/vision-aws.py b/examples/vision/vision-aws.py index 47d2b7970..8905f1e9f 100644 --- a/examples/vision/vision-aws.py +++ b/examples/vision/vision-aws.py @@ -49,10 +49,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), diff --git a/examples/vision/vision-gemini-flash.py b/examples/vision/vision-gemini-flash.py index 248bc08a7..b94942766 100644 --- a/examples/vision/vision-gemini-flash.py +++ b/examples/vision/vision-gemini-flash.py @@ -49,17 +49,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You are also able to describe images.", ), diff --git a/examples/vision/vision-moondream.py b/examples/vision/vision-moondream.py index 6a73de7b3..5b2a3f2e7 100644 --- a/examples/vision/vision-moondream.py +++ b/examples/vision/vision-moondream.py @@ -41,7 +41,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), diff --git a/examples/vision/vision-openai-responses-http.py b/examples/vision/vision-openai-responses-http.py index 866a86d2c..86c63a266 100644 --- a/examples/vision/vision-openai-responses-http.py +++ b/examples/vision/vision-openai-responses-http.py @@ -49,17 +49,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesHttpLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIResponsesHttpLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You are also able to describe images.", ), diff --git a/examples/vision/vision-openai-responses.py b/examples/vision/vision-openai-responses.py index a3c113cb2..d66c2b2ba 100644 --- a/examples/vision/vision-openai-responses.py +++ b/examples/vision/vision-openai-responses.py @@ -49,17 +49,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIResponsesLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You are also able to describe images.", ), diff --git a/examples/vision/vision-openai.py b/examples/vision/vision-openai.py index 8c8af6352..2dfede380 100644 --- a/examples/vision/vision-openai.py +++ b/examples/vision/vision-openai.py @@ -49,17 +49,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. You are also able to describe images.", ), diff --git a/examples/voice/voice-aicoustics.py b/examples/voice/voice-aicoustics.py index 16b850052..39110de88 100644 --- a/examples/voice/voice-aicoustics.py +++ b/examples/voice/voice-aicoustics.py @@ -73,17 +73,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-assemblyai-turn-detection.py b/examples/voice/voice-assemblyai-turn-detection.py index 64d257675..26662872e 100644 --- a/examples/voice/voice-assemblyai-turn-detection.py +++ b/examples/voice/voice-assemblyai-turn-detection.py @@ -91,7 +91,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = AssemblyAISTTService( - api_key=os.getenv("ASSEMBLYAI_API_KEY"), + api_key=os.environ["ASSEMBLYAI_API_KEY"], vad_force_turn_endpoint=False, # Use AssemblyAI's built-in turn detection settings=AssemblyAISTTService.Settings( model="u3-rt-pro", @@ -106,14 +106,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-assemblyai.py b/examples/voice/voice-assemblyai.py index 6bcae005a..826b4b020 100644 --- a/examples/voice/voice-assemblyai.py +++ b/examples/voice/voice-assemblyai.py @@ -54,18 +54,18 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = AssemblyAISTTService( - api_key=os.getenv("ASSEMBLYAI_API_KEY"), + api_key=os.environ["ASSEMBLYAI_API_KEY"], ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-asyncai-http.py b/examples/voice/voice-asyncai-http.py index b861b32ad..480ff9bfd 100644 --- a/examples/voice/voice-asyncai-http.py +++ b/examples/voice/voice-asyncai-http.py @@ -56,7 +56,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Create an HTTP session async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = AsyncAIHttpTTSService( api_key=os.getenv("ASYNCAI_API_KEY", ""), @@ -67,7 +67,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-asyncai.py b/examples/voice/voice-asyncai.py index 55daf5e93..46240542c 100644 --- a/examples/voice/voice-asyncai.py +++ b/examples/voice/voice-asyncai.py @@ -53,7 +53,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = AsyncAITTSService( api_key=os.getenv("ASYNCAI_API_KEY", ""), @@ -63,7 +63,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-azure-http.py b/examples/voice/voice-azure-http.py index 6d8206910..c3872cbfb 100644 --- a/examples/voice/voice-azure-http.py +++ b/examples/voice/voice-azure-http.py @@ -53,18 +53,18 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = AzureSTTService( - api_key=os.getenv("AZURE_SPEECH_API_KEY"), + api_key=os.environ["AZURE_SPEECH_API_KEY"], region=os.getenv("AZURE_SPEECH_REGION"), ) tts = AzureHttpTTSService( - api_key=os.getenv("AZURE_SPEECH_API_KEY"), - region=os.getenv("AZURE_SPEECH_REGION"), + api_key=os.environ["AZURE_SPEECH_API_KEY"], + region=os.environ["AZURE_SPEECH_REGION"], ) llm = AzureLLMService( - api_key=os.getenv("AZURE_CHATGPT_API_KEY"), - endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + api_key=os.environ["AZURE_CHATGPT_API_KEY"], + endpoint=os.environ["AZURE_CHATGPT_ENDPOINT"], settings=AzureLLMService.Settings( model=os.getenv("AZURE_CHATGPT_MODEL"), system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/voice/voice-azure.py b/examples/voice/voice-azure.py index a7977b04b..708c96e18 100644 --- a/examples/voice/voice-azure.py +++ b/examples/voice/voice-azure.py @@ -53,18 +53,18 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = AzureSTTService( - api_key=os.getenv("AZURE_SPEECH_API_KEY"), + api_key=os.environ["AZURE_SPEECH_API_KEY"], region=os.getenv("AZURE_SPEECH_REGION"), ) tts = AzureTTSService( - api_key=os.getenv("AZURE_SPEECH_API_KEY"), - region=os.getenv("AZURE_SPEECH_REGION"), + api_key=os.environ["AZURE_SPEECH_API_KEY"], + region=os.environ["AZURE_SPEECH_REGION"], ) llm = AzureLLMService( - api_key=os.getenv("AZURE_CHATGPT_API_KEY"), - endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + api_key=os.environ["AZURE_CHATGPT_API_KEY"], + endpoint=os.environ["AZURE_CHATGPT_ENDPOINT"], settings=AzureLLMService.Settings( model=os.getenv("AZURE_CHATGPT_MODEL"), system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", diff --git a/examples/voice/voice-camb.py b/examples/voice/voice-camb.py index 90b6cabd4..a84b3c3ec 100644 --- a/examples/voice/voice-camb.py +++ b/examples/voice/voice-camb.py @@ -52,17 +52,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info("Starting Camb AI TTS bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CambTTSService( - api_key=os.getenv("CAMB_API_KEY"), + api_key=os.environ["CAMB_API_KEY"], settings=CambTTSService.Settings( model="mars-flash", ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful voice assistant powered by Camb AI text-to-speech. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Keep responses concise.", ), diff --git a/examples/voice/voice-cartesia-http.py b/examples/voice/voice-cartesia-http.py index 15fe51142..f2f02ec7d 100644 --- a/examples/voice/voice-cartesia-http.py +++ b/examples/voice/voice-cartesia-http.py @@ -54,10 +54,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = CartesiaSTTService(api_key=os.getenv("CARTESIA_API_KEY")) + stt = CartesiaSTTService(api_key=os.environ["CARTESIA_API_KEY"]) tts = CartesiaHttpTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], aiohttp_session=session, settings=CartesiaHttpTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady @@ -65,7 +65,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-cartesia.py b/examples/voice/voice-cartesia.py index 0489663e7..2a399b954 100644 --- a/examples/voice/voice-cartesia.py +++ b/examples/voice/voice-cartesia.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = CartesiaSTTService(api_key=os.getenv("CARTESIA_API_KEY")) + stt = CartesiaSTTService(api_key=os.environ["CARTESIA_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-deepgram-flux-sagemaker.py b/examples/voice/voice-deepgram-flux-sagemaker.py index c203ed43a..c0426ab15 100644 --- a/examples/voice/voice-deepgram-flux-sagemaker.py +++ b/examples/voice/voice-deepgram-flux-sagemaker.py @@ -59,8 +59,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # - AWS credentials configured (via environment variables or AWS CLI) # - A deployed SageMaker endpoint with Deepgram Flux model stt = DeepgramFluxSageMakerSTTService( - endpoint_name=os.getenv("SAGEMAKER_STT_ENDPOINT_NAME"), - region=os.getenv("AWS_REGION"), + endpoint_name=os.environ["SAGEMAKER_STT_ENDPOINT_NAME"], + region=os.environ["AWS_REGION"], settings=DeepgramFluxSageMakerSTTService.Settings( min_confidence=0.3, ), @@ -71,8 +71,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # - AWS credentials configured (via environment variables or AWS CLI) # - A deployed SageMaker endpoint with Deepgram TTS model tts = DeepgramSageMakerTTSService( - endpoint_name=os.getenv("SAGEMAKER_TTS_ENDPOINT_NAME"), - region=os.getenv("AWS_REGION"), + endpoint_name=os.environ["SAGEMAKER_TTS_ENDPOINT_NAME"], + region=os.environ["AWS_REGION"], settings=DeepgramSageMakerTTSService.Settings( voice="aura-2-andromeda-en", ), diff --git a/examples/voice/voice-deepgram-flux.py b/examples/voice/voice-deepgram-flux.py index 76a5bb783..7e9ec3938 100644 --- a/examples/voice/voice-deepgram-flux.py +++ b/examples/voice/voice-deepgram-flux.py @@ -55,21 +55,21 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = DeepgramFluxSTTService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], settings=DeepgramFluxSTTService.Settings( min_confidence=0.3, ), ) tts = DeepgramTTSService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], settings=DeepgramTTSService.Settings( voice="aura-2-andromeda-en", ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-deepgram-http.py b/examples/voice/voice-deepgram-http.py index c81b2cd2f..d53255ca0 100644 --- a/examples/voice/voice-deepgram-http.py +++ b/examples/voice/voice-deepgram-http.py @@ -55,10 +55,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = DeepgramHttpTTSService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], settings=DeepgramHttpTTSService.Settings( voice="aura-2-andromeda-en", ), @@ -66,7 +66,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-deepgram-sagemaker.py b/examples/voice/voice-deepgram-sagemaker.py index e823f18f9..30e861518 100644 --- a/examples/voice/voice-deepgram-sagemaker.py +++ b/examples/voice/voice-deepgram-sagemaker.py @@ -58,8 +58,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # - AWS credentials configured (via environment variables or AWS CLI) # - A deployed SageMaker endpoint with Deepgram model stt = DeepgramSageMakerSTTService( - endpoint_name=os.getenv("SAGEMAKER_STT_ENDPOINT_NAME"), - region=os.getenv("AWS_REGION"), + endpoint_name=os.environ["SAGEMAKER_STT_ENDPOINT_NAME"], + region=os.environ["AWS_REGION"], ) # Initialize Deepgram SageMaker TTS Service @@ -67,8 +67,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # - AWS credentials configured (via environment variables or AWS CLI) # - A deployed SageMaker endpoint with Deepgram TTS model tts = DeepgramSageMakerTTSService( - endpoint_name=os.getenv("SAGEMAKER_TTS_ENDPOINT_NAME"), - region=os.getenv("AWS_REGION"), + endpoint_name=os.environ["SAGEMAKER_TTS_ENDPOINT_NAME"], + region=os.environ["AWS_REGION"], settings=DeepgramSageMakerTTSService.Settings( voice="aura-2-andromeda-en", ), diff --git a/examples/voice/voice-deepgram-vad.py b/examples/voice/voice-deepgram-vad.py deleted file mode 100644 index bbdb01efe..000000000 --- a/examples/voice/voice-deepgram-vad.py +++ /dev/null @@ -1,133 +0,0 @@ -# -# Copyright (c) 2024-2026, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - - -import os - -from dotenv import load_dotenv -from loguru import logger - -from pipecat.frames.frames import LLMRunFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import ( - LLMContextAggregatorPair, - LLMUserAggregatorParams, -) -from pipecat.runner.types import RunnerArguments -from pipecat.runner.utils import create_transport -from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.services.deepgram.tts import DeepgramTTSService -from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.transports.daily.transport import DailyParams -from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams -from pipecat.turns.user_turn_strategies import ExternalUserTurnStrategies - -load_dotenv(override=True) - - -# We use lambdas to defer transport parameter creation until the transport -# type is selected at runtime. -transport_params = { - "daily": lambda: DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - ), - "twilio": lambda: FastAPIWebsocketParams( - audio_in_enabled=True, - audio_out_enabled=True, - ), - "webrtc": lambda: TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - ), -} - - -async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - logger.info(f"Starting bot") - - stt = DeepgramSTTService( - api_key=os.getenv("DEEPGRAM_API_KEY"), - settings=DeepgramSTTService.Settings( - vad_events=True, - utterance_end_ms="1000", - ), - ) - - tts = DeepgramTTSService( - api_key=os.getenv("DEEPGRAM_API_KEY"), - settings=DeepgramTTSService.Settings( - voice="aura-2-andromeda-en", - ), - ) - - llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), - settings=OpenAILLMService.Settings( - system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", - ), - ) - - context = LLMContext() - user_aggregator, assistant_aggregator = LLMContextAggregatorPair( - context, - user_params=LLMUserAggregatorParams(user_turn_strategies=ExternalUserTurnStrategies()), - ) - - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, # STT - user_aggregator, # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - assistant_aggregator, # Assistant spoken responses - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - enable_usage_metrics=True, - ), - idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - ) - - @transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - logger.info(f"Client connected") - # Kick off the conversation. - context.add_message( - {"role": "developer", "content": "Please introduce yourself to the user."} - ) - await task.queue_frames([LLMRunFrame()]) - - @transport.event_handler("on_client_disconnected") - async def on_client_disconnected(transport, client): - logger.info(f"Client disconnected") - await task.cancel() - - runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) - - await runner.run(task) - - -async def bot(runner_args: RunnerArguments): - """Main bot entry point compatible with Pipecat Cloud.""" - transport = await create_transport(runner_args, transport_params) - await run_bot(transport, runner_args) - - -if __name__ == "__main__": - from pipecat.runner.run import main - - main() diff --git a/examples/voice/voice-deepgram.py b/examples/voice/voice-deepgram.py index bebde3fe3..323fd34a3 100644 --- a/examples/voice/voice-deepgram.py +++ b/examples/voice/voice-deepgram.py @@ -53,17 +53,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = DeepgramTTSService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], settings=DeepgramTTSService.Settings( voice="aura-2-andromeda-en", ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-elevenlabs-http.py b/examples/voice/voice-elevenlabs-http.py index b9e35ec4f..87e8dc2ca 100644 --- a/examples/voice/voice-elevenlabs-http.py +++ b/examples/voice/voice-elevenlabs-http.py @@ -57,7 +57,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Create an HTTP session async with aiohttp.ClientSession() as session: stt = ElevenLabsSTTService( - api_key=os.getenv("ELEVENLABS_API_KEY"), + api_key=os.environ["ELEVENLABS_API_KEY"], aiohttp_session=session, ) @@ -70,7 +70,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-elevenlabs.py b/examples/voice/voice-elevenlabs.py index ebdfdde72..8f61bb665 100644 --- a/examples/voice/voice-elevenlabs.py +++ b/examples/voice/voice-elevenlabs.py @@ -53,7 +53,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = ElevenLabsRealtimeSTTService(api_key=os.getenv("ELEVENLABS_API_KEY")) + stt = ElevenLabsRealtimeSTTService(api_key=os.environ["ELEVENLABS_API_KEY"]) tts = ElevenLabsTTSService( api_key=os.getenv("ELEVENLABS_API_KEY", ""), @@ -63,7 +63,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-fal.py b/examples/voice/voice-fal.py index 5ce43bbe6..d4b1e4a52 100644 --- a/examples/voice/voice-fal.py +++ b/examples/voice/voice-fal.py @@ -61,14 +61,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-fish.py b/examples/voice/voice-fish.py index 5519b6c4f..0ff8503c8 100644 --- a/examples/voice/voice-fish.py +++ b/examples/voice/voice-fish.py @@ -53,17 +53,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = FishAudioTTSService( - api_key=os.getenv("FISH_API_KEY"), + api_key=os.environ["FISH_API_KEY"], settings=FishAudioTTSService.Settings( voice="4ce7e917cedd4bc2bb2e6ff3a46acaa1", # Barack Obama ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-gladia-vad.py b/examples/voice/voice-gladia-vad.py index f29930c3e..c0eea8841 100644 --- a/examples/voice/voice-gladia-vad.py +++ b/examples/voice/voice-gladia-vad.py @@ -55,9 +55,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") + region = os.getenv("GLADIA_REGION", "us-west") + assert region in ("us-west", "eu-west"), f"Invalid GLADIA_REGION: {region}" + stt = GladiaSTTService( - api_key=os.getenv("GLADIA_API_KEY", ""), - region=os.getenv("GLADIA_REGION"), + api_key=os.environ["GLADIA_API_KEY"], + region=region, settings=GladiaSTTService.Settings( language_config=LanguageConfig( languages=[Language.EN], diff --git a/examples/voice/voice-gladia.py b/examples/voice/voice-gladia.py index 3f417bf5a..a6a909a9a 100644 --- a/examples/voice/voice-gladia.py +++ b/examples/voice/voice-gladia.py @@ -54,9 +54,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") + region = os.getenv("GLADIA_REGION", "us-west") + assert region in ("us-west", "eu-west"), f"Invalid GLADIA_REGION: {region}" + stt = GladiaSTTService( - api_key=os.getenv("GLADIA_API_KEY", ""), - region=os.getenv("GLADIA_REGION"), + api_key=os.environ["GLADIA_API_KEY"], + region=region, settings=GladiaSTTService.Settings( language_config=LanguageConfig( languages=[Language.EN], diff --git a/examples/voice/voice-google-audio-in.py b/examples/voice/voice-google-audio-in.py index fa314afed..f6a350411 100644 --- a/examples/voice/voice-google-audio-in.py +++ b/examples/voice/voice-google-audio-in.py @@ -215,7 +215,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( model="gemini-2.5-flash", system_instruction=system_message, @@ -230,7 +230,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): language=Language.EN_US, ), params=GoogleTTSService.InputParams(language=Language.EN_US), - credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + credentials=os.environ["GOOGLE_TEST_CREDENTIALS"], ) context = LLMContext() diff --git a/examples/voice/voice-google-gemini-tts.py b/examples/voice/voice-google-gemini-tts.py index 7b995a564..8aa7e3c39 100644 --- a/examples/voice/voice-google-gemini-tts.py +++ b/examples/voice/voice-google-gemini-tts.py @@ -57,11 +57,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): settings=GoogleSTTService.Settings( languages=[Language.EN_US], ), - credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + credentials=os.environ["GOOGLE_TEST_CREDENTIALS"], ) tts = GeminiTTSService( - credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + credentials=os.environ["GOOGLE_TEST_CREDENTIALS"], settings=GeminiTTSService.Settings( model="gemini-2.5-flash-tts", voice="Charon", @@ -71,7 +71,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], model="gemini-2.5-flash", settings=GoogleLLMService.Settings( system_instruction="""You are a helpful assistant in a voice conversation. diff --git a/examples/voice/voice-google-http.py b/examples/voice/voice-google-http.py index d961f2d8d..0e642b8ca 100644 --- a/examples/voice/voice-google-http.py +++ b/examples/voice/voice-google-http.py @@ -59,7 +59,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Add model to use a specific model # model="chirp_3", ), - credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + credentials=os.environ["GOOGLE_TEST_CREDENTIALS"], location="us", ) @@ -68,11 +68,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): voice="en-US-Chirp3-HD-Charon", language=Language.EN_US, ), - credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + credentials=os.environ["GOOGLE_TEST_CREDENTIALS"], ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( model="gemini-2.5-flash", # force a certain amount of thinking if you want it diff --git a/examples/voice/voice-google-image.py b/examples/voice/voice-google-image.py index 80f115390..52949b357 100644 --- a/examples/voice/voice-google-image.py +++ b/examples/voice/voice-google-image.py @@ -67,14 +67,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = GoogleSTTService( - credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + credentials=os.environ["GOOGLE_TEST_CREDENTIALS"], settings=GoogleSTTService.Settings( languages=[Language.EN_US], ), ) tts = GoogleTTSService( - credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + credentials=os.environ["GOOGLE_TEST_CREDENTIALS"], settings=GoogleTTSService.Settings( voice="en-US-Chirp3-HD-Charon", language=Language.EN_US, @@ -82,7 +82,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( model="gemini-2.5-flash-image", # model="gemini-3-pro-image-preview", # A more powerful model, but slower, diff --git a/examples/voice/voice-google.py b/examples/voice/voice-google.py index 10684b8cc..9f6a43874 100644 --- a/examples/voice/voice-google.py +++ b/examples/voice/voice-google.py @@ -59,7 +59,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Add model to use a specific model # model="chirp_3", ), - credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + credentials=os.environ["GOOGLE_TEST_CREDENTIALS"], location="us", ) @@ -68,11 +68,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): voice="en-US-Chirp3-HD-Charon", language=Language.EN_US, ), - credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + credentials=os.environ["GOOGLE_TEST_CREDENTIALS"], ) llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), + api_key=os.environ["GOOGLE_API_KEY"], settings=GoogleLLMService.Settings( model="gemini-2.5-flash", # force a certain amount of thinking if you want it diff --git a/examples/voice/voice-gradium.py b/examples/voice/voice-gradium.py index 3573d0b39..89573f48a 100644 --- a/examples/voice/voice-gradium.py +++ b/examples/voice/voice-gradium.py @@ -53,7 +53,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = GradiumSTTService( - api_key=os.getenv("GRADIUM_API_KEY"), + api_key=os.environ["GRADIUM_API_KEY"], api_endpoint_base_url="wss://us.api.gradium.ai/api/speech/asr", settings=GradiumSTTService.Settings( language=Language.EN, @@ -61,7 +61,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = GradiumTTSService( - api_key=os.getenv("GRADIUM_API_KEY"), + api_key=os.environ["GRADIUM_API_KEY"], url="wss://us.api.gradium.ai/api/speech/tts", settings=GradiumTTSService.Settings( voice="YTpq7expH9539ERJ", @@ -69,7 +69,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-groq.py b/examples/voice/voice-groq.py index 7ffa8b06b..9b1b34da4 100644 --- a/examples/voice/voice-groq.py +++ b/examples/voice/voice-groq.py @@ -52,17 +52,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = GroqSTTService(api_key=os.getenv("GROQ_API_KEY")) + stt = GroqSTTService(api_key=os.environ["GROQ_API_KEY"]) llm = GroqLLMService( - api_key=os.getenv("GROQ_API_KEY"), + api_key=os.environ["GROQ_API_KEY"], settings=GroqLLMService.Settings( model="llama-3.1-8b-instant", system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), ) - tts = GroqTTSService(api_key=os.getenv("GROQ_API_KEY")) + tts = GroqTTSService(api_key=os.environ["GROQ_API_KEY"]) context = LLMContext() user_aggregator, assistant_aggregator = LLMContextAggregatorPair( diff --git a/examples/voice/voice-hume.py b/examples/voice/voice-hume.py index 2dbeeb924..bda68d6f9 100644 --- a/examples/voice/voice-hume.py +++ b/examples/voice/voice-hume.py @@ -54,7 +54,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = HumeTTSService( api_key=os.getenv("HUME_API_KEY"), @@ -65,7 +65,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-inworld-http.py b/examples/voice/voice-inworld-http.py index 3d06a055a..2a388f62b 100644 --- a/examples/voice/voice-inworld-http.py +++ b/examples/voice/voice-inworld-http.py @@ -53,7 +53,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info("Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = InworldHttpTTSService( api_key=os.getenv("INWORLD_API_KEY", ""), @@ -66,7 +66,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful AI demonstrating Inworld AI's TTS. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a friendly and helpful way.", ), diff --git a/examples/voice/voice-inworld.py b/examples/voice/voice-inworld.py index 7db6edbee..b8603efc9 100644 --- a/examples/voice/voice-inworld.py +++ b/examples/voice/voice-inworld.py @@ -50,7 +50,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info("Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = InworldTTSService( api_key=os.getenv("INWORLD_API_KEY", ""), @@ -61,7 +61,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful AI demonstrating Inworld AI's TTS. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a friendly and helpful way.", ), diff --git a/examples/voice/voice-kokoro.py b/examples/voice/voice-kokoro.py index c55c2ba08..fe74589dc 100644 --- a/examples/voice/voice-kokoro.py +++ b/examples/voice/voice-kokoro.py @@ -52,7 +52,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = KokoroTTSService( settings=KokoroTTSService.Settings( @@ -61,7 +61,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-krisp-viva.py b/examples/voice/voice-krisp-viva.py index 5fdefd2e1..fa6498616 100644 --- a/examples/voice/voice-krisp-viva.py +++ b/examples/voice/voice-krisp-viva.py @@ -4,20 +4,24 @@ # SPDX-License-Identifier: BSD 2-Clause License # -"""Interruptible bot with Krisp VIVA noise filtering and turn detection. +"""Interruptible bot with Krisp VIVA noise filtering, turn detection, and IP. This example demonstrates a conversational bot with: - Krisp VIVA noise reduction on incoming audio -- Krisp VIVA Turn detection for natural interruptions +- Krisp VIVA Turn detection for end-of-turn +- Krisp Interruption Prediction (IP) to filter backchannels from real interruptions - Voice activity detection (VAD) Required environment variables: - KRISP_VIVA_FILTER_MODEL_PATH: Path to the Krisp noise filter model file (.kef) - KRISP_VIVA_TURN_MODEL_PATH: Path to the Krisp turn detection model file (.kef) -- DEEPGRAM_API_KEY: Deepgram API key for STT/TTS +- KRISP_VIVA_IP_MODEL_PATH: Path to the Krisp IP model file (.kef) +- DEEPGRAM_API_KEY: Deepgram API key for STT +- CARTESIA_API_KEY: Cartesia API key for TTS - OPENAI_API_KEY: OpenAI API key for LLM Optional environment variables: +- KRISP_VIVA_API_KEY: Krisp SDK API key (or set in code) - KRISP_NOISE_SUPPRESSION_LEVEL: Noise suppression level 0-100 (default: 100) Higher values = more aggressive noise reduction """ @@ -49,31 +53,30 @@ from pipecat.services.openai.llm import OpenAILLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams +from pipecat.turns.user_start import ( + KrispVivaIPUserTurnStartStrategy, + TranscriptionUserTurnStartStrategy, +) from pipecat.turns.user_stop import TurnAnalyzerUserTurnStopStrategy from pipecat.turns.user_turn_strategies import UserTurnStrategies load_dotenv(override=True) -# We use lambdas to defer transport parameter creation until the transport -# type is selected at runtime. - -krisp_viva_filter = KrispVivaFilter() - transport_params = { "daily": lambda: DailyParams( audio_in_enabled=True, audio_out_enabled=True, - audio_in_filter=krisp_viva_filter, + audio_in_filter=KrispVivaFilter(), ), "twilio": lambda: FastAPIWebsocketParams( audio_in_enabled=True, audio_out_enabled=True, - audio_in_filter=krisp_viva_filter, + audio_in_filter=KrispVivaFilter(), ), "webrtc": lambda: TransportParams( audio_in_enabled=True, audio_out_enabled=True, - audio_in_filter=krisp_viva_filter, + audio_in_filter=KrispVivaFilter(), ), } @@ -81,17 +84,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), @@ -102,7 +105,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): context, user_params=LLMUserAggregatorParams( user_turn_strategies=UserTurnStrategies( - stop=[TurnAnalyzerUserTurnStopStrategy(turn_analyzer=KrispVivaTurn())] + start=[ + KrispVivaIPUserTurnStartStrategy(threshold=0.5), + TranscriptionUserTurnStartStrategy(), + ], + stop=[TurnAnalyzerUserTurnStopStrategy(turn_analyzer=KrispVivaTurn())], ), vad_analyzer=SileroVADAnalyzer(), # or KrispVivaVadAnalyzer ), diff --git a/examples/voice/voice-langchain.py b/examples/voice/voice-langchain.py index 8ab17aa79..19fda0c91 100644 --- a/examples/voice/voice-langchain.py +++ b/examples/voice/voice-langchain.py @@ -16,7 +16,7 @@ from langchain_openai import ChatOpenAI from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import LLMMessagesUpdateFrame +from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -67,10 +67,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), @@ -129,8 +129,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # An `LLMContextFrame` will be picked up by the LangchainProcessor using # only the content of the last message to inject it in the prompt defined # above. So no role is required here. - messages = [({"content": "Please briefly introduce yourself to the user."})] - await task.queue_frames([LLMMessagesUpdateFrame(messages, run_llm=True)]) + context.add_message( + {"role": "developer", "content": "Please briefly introduce yourself to the user."} + ) + await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): diff --git a/examples/voice/voice-lmnt.py b/examples/voice/voice-lmnt.py index 41e42a961..6faa4c31c 100644 --- a/examples/voice/voice-lmnt.py +++ b/examples/voice/voice-lmnt.py @@ -52,17 +52,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = LmntTTSService( - api_key=os.getenv("LMNT_API_KEY"), + api_key=os.environ["LMNT_API_KEY"], settings=LmntTTSService.Settings( voice="morgan", ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-minimax.py b/examples/voice/voice-minimax.py index 92fda761d..9e4291af7 100644 --- a/examples/voice/voice-minimax.py +++ b/examples/voice/voice-minimax.py @@ -57,7 +57,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Create an HTTP session async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = MiniMaxHttpTTSService( api_key=os.getenv("MINIMAX_API_KEY", ""), @@ -69,7 +69,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-mistral.py b/examples/voice/voice-mistral.py index 440039b65..4664caf22 100644 --- a/examples/voice/voice-mistral.py +++ b/examples/voice/voice-mistral.py @@ -53,17 +53,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = MistralSTTService(api_key=os.getenv("MISTRAL_API_KEY")) + stt = MistralSTTService(api_key=os.environ["MISTRAL_API_KEY"]) tts = MistralTTSService( - api_key=os.getenv("MISTRAL_API_KEY"), + api_key=os.environ["MISTRAL_API_KEY"], settings=MistralTTSService.Settings( voice="c69964a6-ab8b-4f8a-9465-ec0925096ec8", ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-neuphonic-http.py b/examples/voice/voice-neuphonic-http.py index 6392d4606..9cbcc8ede 100644 --- a/examples/voice/voice-neuphonic-http.py +++ b/examples/voice/voice-neuphonic-http.py @@ -56,10 +56,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Create an HTTP session async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = NeuphonicHttpTTSService( - api_key=os.getenv("NEUPHONIC_API_KEY"), + api_key=os.environ["NEUPHONIC_API_KEY"], settings=NeuphonicHttpTTSService.Settings( voice="fc854436-2dac-4d21-aa69-ae17b54e98eb", # Emily ), @@ -67,7 +67,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-neuphonic.py b/examples/voice/voice-neuphonic.py index a0aaf82fa..1ea90fd51 100644 --- a/examples/voice/voice-neuphonic.py +++ b/examples/voice/voice-neuphonic.py @@ -52,17 +52,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = NeuphonicTTSService( - api_key=os.getenv("NEUPHONIC_API_KEY"), + api_key=os.environ["NEUPHONIC_API_KEY"], settings=NeuphonicTTSService.Settings( voice="fc854436-2dac-4d21-aa69-ae17b54e98eb", # Emily ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-nvidia.py b/examples/voice/voice-nvidia.py index b308f71a2..7f84134d3 100644 --- a/examples/voice/voice-nvidia.py +++ b/examples/voice/voice-nvidia.py @@ -52,17 +52,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = NvidiaSTTService(api_key=os.getenv("NVIDIA_API_KEY")) + stt = NvidiaSTTService(api_key=os.environ["NVIDIA_API_KEY"]) llm = NvidiaLLMService( - api_key=os.getenv("NVIDIA_API_KEY"), + api_key=os.environ["NVIDIA_API_KEY"], settings=NvidiaLLMService.Settings( model="meta/llama-3.3-70b-instruct", system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), ) - tts = NvidiaTTSService(api_key=os.getenv("NVIDIA_API_KEY")) + tts = NvidiaTTSService(api_key=os.environ["NVIDIA_API_KEY"]) context = LLMContext() user_aggregator, assistant_aggregator = LLMContextAggregatorPair( diff --git a/examples/voice/voice-openai-http.py b/examples/voice/voice-openai-http.py index 19f1a3e8a..1483acb69 100644 --- a/examples/voice/voice-openai-http.py +++ b/examples/voice/voice-openai-http.py @@ -53,7 +53,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = OpenAISTTService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAISTTService.Settings( model="gpt-4o-transcribe", prompt="Expect words related to dogs, such as breed names.", @@ -61,14 +61,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = OpenAITTSService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAITTSService.Settings( voice="ballad", ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are very knowledgable about dogs. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", ), diff --git a/examples/voice/voice-openai-responses-http.py b/examples/voice/voice-openai-responses-http.py index 6960bf7b2..b3f49cbe1 100644 --- a/examples/voice/voice-openai-responses-http.py +++ b/examples/voice/voice-openai-responses-http.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesHttpLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIResponsesHttpLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-openai-responses.py b/examples/voice/voice-openai-responses.py index baae3754a..d1ecc0946 100644 --- a/examples/voice/voice-openai-responses.py +++ b/examples/voice/voice-openai-responses.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAIResponsesLLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIResponsesLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-openai.py b/examples/voice/voice-openai.py index b749ce27a..8f162671f 100644 --- a/examples/voice/voice-openai.py +++ b/examples/voice/voice-openai.py @@ -54,7 +54,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = OpenAIRealtimeSTTService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAIRealtimeSTTService.Settings( model="gpt-4o-transcribe", prompt="Expect words related to dogs, such as breed names.", @@ -63,14 +63,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = OpenAITTSService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAITTSService.Settings( voice="ballad", ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are very knowledgable about dogs. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", ), diff --git a/examples/voice/voice-piper.py b/examples/voice/voice-piper.py index 617223ceb..dffdd4d34 100644 --- a/examples/voice/voice-piper.py +++ b/examples/voice/voice-piper.py @@ -52,7 +52,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = PiperTTSService( settings=PiperTTSService.Settings( @@ -61,7 +61,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-resemble.py b/examples/voice/voice-resemble.py index c27ca9255..f3c2a1336 100644 --- a/examples/voice/voice-resemble.py +++ b/examples/voice/voice-resemble.py @@ -51,17 +51,17 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = ResembleAITTSService( - api_key=os.getenv("RESEMBLE_API_KEY"), + api_key=os.environ["RESEMBLE_API_KEY"], settings=ResembleAITTSService.Settings( - voice=os.getenv("RESEMBLE_VOICE_UUID"), + voice=os.environ["RESEMBLE_VOICE_UUID"], ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-rime-http.py b/examples/voice/voice-rime-http.py index 165735d6e..72bc63ca7 100644 --- a/examples/voice/voice-rime-http.py +++ b/examples/voice/voice-rime-http.py @@ -56,7 +56,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Create an HTTP session async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = RimeHttpTTSService( api_key=os.getenv("RIME_API_KEY", ""), @@ -69,7 +69,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-rime.py b/examples/voice/voice-rime.py index 07ea429c7..bd1f50dbc 100644 --- a/examples/voice/voice-rime.py +++ b/examples/voice/voice-rime.py @@ -52,7 +52,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = RimeTTSService( api_key=os.getenv("RIME_API_KEY", ""), @@ -62,7 +62,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-sarvam-http.py b/examples/voice/voice-sarvam-http.py index bca27e051..b70f44e2a 100644 --- a/examples/voice/voice-sarvam-http.py +++ b/examples/voice/voice-sarvam-http.py @@ -58,14 +58,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Create an HTTP session async with aiohttp.ClientSession() as session: stt = SarvamSTTService( - api_key=os.getenv("SARVAM_API_KEY"), + api_key=os.environ["SARVAM_API_KEY"], settings=SarvamSTTService.Settings( model="saarika:v2.5", ), ) tts = SarvamHttpTTSService( - api_key=os.getenv("SARVAM_API_KEY"), + api_key=os.environ["SARVAM_API_KEY"], aiohttp_session=session, settings=SarvamHttpTTSService.Settings( language=Language.EN_IN, @@ -73,7 +73,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = SarvamLLMService( - api_key=os.getenv("SARVAM_API_KEY"), + api_key=os.environ["SARVAM_API_KEY"], settings=SarvamLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-sarvam.py b/examples/voice/voice-sarvam.py index feda4fe38..5c646d505 100644 --- a/examples/voice/voice-sarvam.py +++ b/examples/voice/voice-sarvam.py @@ -53,21 +53,21 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = SarvamSTTService( - api_key=os.getenv("SARVAM_API_KEY"), + api_key=os.environ["SARVAM_API_KEY"], settings=SarvamSTTService.Settings( model="saaras:v3", ), ) tts = SarvamTTSService( - api_key=os.getenv("SARVAM_API_KEY"), + api_key=os.environ["SARVAM_API_KEY"], settings=SarvamTTSService.Settings( model="bulbul:v3", voice="shubh", ), ) llm = SarvamLLMService( - api_key=os.getenv("SARVAM_API_KEY"), + api_key=os.environ["SARVAM_API_KEY"], settings=SarvamLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-smallest.py b/examples/voice/voice-smallest.py index f1b3dcf43..b5b82802a 100644 --- a/examples/voice/voice-smallest.py +++ b/examples/voice/voice-smallest.py @@ -52,21 +52,21 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = SmallestSTTService( - api_key=os.getenv("SMALLEST_API_KEY"), + api_key=os.environ["SMALLEST_API_KEY"], settings=SmallestSTTService.Settings( language=Language.EN, ), ) tts = SmallestTTSService( - api_key=os.getenv("SMALLEST_API_KEY"), + api_key=os.environ["SMALLEST_API_KEY"], settings=SmallestTTSService.Settings( voice="sophia", ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", ), diff --git a/examples/voice/voice-soniox.py b/examples/voice/voice-soniox.py index 857fd19e8..b89106d14 100644 --- a/examples/voice/voice-soniox.py +++ b/examples/voice/voice-soniox.py @@ -52,7 +52,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") stt = SonioxSTTService( - api_key=os.getenv("SONIOX_API_KEY"), + api_key=os.environ["SONIOX_API_KEY"], settings=SonioxSTTService.Settings( # Add language hints to use a specific language # Add strict mode to enforce the language hints @@ -62,14 +62,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ), ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-speechmatics-vad.py b/examples/voice/voice-speechmatics-vad.py index bcceb4dea..b21bf7156 100644 --- a/examples/voice/voice-speechmatics-vad.py +++ b/examples/voice/voice-speechmatics-vad.py @@ -91,7 +91,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: stt = SpeechmaticsSTTService( - api_key=os.getenv("SPEECHMATICS_API_KEY"), + api_key=os.environ["SPEECHMATICS_API_KEY"], settings=SpeechmaticsSTTService.Settings( language=Language.EN, turn_detection_mode=SpeechmaticsSTTService.TurnDetectionMode.ADAPTIVE, @@ -102,7 +102,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = SpeechmaticsTTSService( - api_key=os.getenv("SPEECHMATICS_API_KEY"), + api_key=os.environ["SPEECHMATICS_API_KEY"], settings=SpeechmaticsTTSService.Settings( voice="sarah", ), @@ -110,7 +110,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( temperature=0.75, system_instruction="You are a helpful British assistant called Sarah in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Always include punctuation in your responses. Give very short replies - do not give longer replies unless strictly necessary. Respond to what the user said in a concise, funny, creative and helpful way. Use `` tags to identify different speakers - do not use tags in your replies. Do not respond to speakers within `` tags unless explicitly asked to.", diff --git a/examples/voice/voice-speechmatics.py b/examples/voice/voice-speechmatics.py index d619715aa..3f7fa9ac4 100644 --- a/examples/voice/voice-speechmatics.py +++ b/examples/voice/voice-speechmatics.py @@ -74,7 +74,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): async with aiohttp.ClientSession() as session: stt = SpeechmaticsSTTService( - api_key=os.getenv("SPEECHMATICS_API_KEY"), + api_key=os.environ["SPEECHMATICS_API_KEY"], settings=SpeechmaticsSTTService.Settings( language=Language.EN, speaker_active_format="<{speaker_id}>{text}", @@ -82,7 +82,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = SpeechmaticsTTSService( - api_key=os.getenv("SPEECHMATICS_API_KEY"), + api_key=os.environ["SPEECHMATICS_API_KEY"], settings=SpeechmaticsTTSService.Settings( voice="sarah", ), @@ -90,7 +90,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( temperature=0.75, system_instruction="You are a helpful British assistant called Sarah in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Always include punctuation in your responses. Give very short replies - do not give longer replies unless strictly necessary. Respond to what the user said in a concise, funny, creative and helpful way. Use `` tags to identify different speakers - do not use tags in your replies. Do not respond to speakers within `` tags unless explicitly asked to.", diff --git a/examples/voice/voice-xai-http.py b/examples/voice/voice-xai-http.py new file mode 100644 index 000000000..6d0d54e79 --- /dev/null +++ b/examples/voice/voice-xai-http.py @@ -0,0 +1,128 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os + +import aiohttp +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.xai.llm import GrokLLMService +from pipecat.services.xai.tts import XAIHttpTTSService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +# We use lambdas to defer transport parameter creation until the transport +# type is selected at runtime. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + async with aiohttp.ClientSession() as session: + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = XAIHttpTTSService( + api_key=os.getenv("XAI_API_KEY"), + aiohttp_session=session, + settings=XAIHttpTTSService.Settings( + voice="eve", + ), + ) + + llm = GrokLLMService( + api_key=os.getenv("XAI_API_KEY"), + settings=GrokLLMService.Settings( + system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", + ), + ) + + context = LLMContext() + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + user_aggregator, # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + assistant_aggregator, # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + context.add_message( + {"role": "developer", "content": "Please introduce yourself to the user."} + ) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/voice/voice-xai.py b/examples/voice/voice-xai.py index 6d0d54e79..541d66c77 100644 --- a/examples/voice/voice-xai.py +++ b/examples/voice/voice-xai.py @@ -22,9 +22,9 @@ from pipecat.processors.aggregators.llm_response_universal import ( ) from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.xai.llm import GrokLLMService -from pipecat.services.xai.tts import XAIHttpTTSService +from pipecat.services.xai.stt import XAISTTService +from pipecat.services.xai.tts import XAITTSService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams @@ -53,18 +53,18 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = XAISTTService(api_key=os.environ["XAI_API_KEY"]) - tts = XAIHttpTTSService( - api_key=os.getenv("XAI_API_KEY"), + tts = XAITTSService( + api_key=os.environ["XAI_API_KEY"], aiohttp_session=session, - settings=XAIHttpTTSService.Settings( + settings=XAITTSService.Settings( voice="eve", ), ) llm = GrokLLMService( - api_key=os.getenv("XAI_API_KEY"), + api_key=os.environ["XAI_API_KEY"], settings=GrokLLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/examples/voice/voice-xtts.py b/examples/voice/voice-xtts.py index f0bfe01c0..f9d92e45b 100644 --- a/examples/voice/voice-xtts.py +++ b/examples/voice/voice-xtts.py @@ -55,7 +55,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Create an HTTP session async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) tts = XTTSService( aiohttp_session=session, @@ -66,7 +66,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), + api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), diff --git a/pyproject.toml b/pyproject.toml index e121eab84..906f8fff5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -106,7 +106,7 @@ rime = [ "pipecat-ai[websockets-base]" ] runner = [ "python-dotenv>=1.0.0,<2.0.0", "uvicorn>=0.32.0,<1.0.0", "fastapi>=0.115.6,<1", "pipecat-ai-small-webrtc-prebuilt>=2.5.0"] sagemaker = ["aws_sdk_sagemaker_runtime_http2; python_version>='3.12'"] sambanova = [] -sarvam = [ "sarvamai==0.1.26", "pipecat-ai[websockets-base]" ] +sarvam = [ "sarvamai==0.1.28", "pipecat-ai[websockets-base]" ] sentry = [ "sentry-sdk>=2.28.0,<3" ] silero = [] simli = [ "simli-ai~=2.0.1"] @@ -123,7 +123,7 @@ webrtc = [ "aiortc>=1.14.0,<2", "opencv-python>=4.11.0.86,<5" ] websocket = [ "pipecat-ai[websockets-base]", "fastapi>=0.115.6,<1" ] websockets-base = [ "websockets>=13.1,<16.0" ] whisper = [ "faster-whisper~=1.2.1" ] -xai = [] +xai = [ "pipecat-ai[websockets-base]" ] [dependency-groups] dev = [ diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 000000000..74682c3dc --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,145 @@ +{ + "typeCheckingMode": "basic", + "pythonVersion": "3.11", + "pythonPlatform": "All", + "include": ["scripts", "src/pipecat"], + "exclude": ["**/*_pb2.py", "**/__pycache__"], + "ignore": [ + "tests", + "src/pipecat/adapters/base_llm_adapter.py", + "src/pipecat/adapters/schemas/direct_function.py", + "src/pipecat/adapters/schemas/tools_schema.py", + "src/pipecat/adapters/services/anthropic_adapter.py", + "src/pipecat/adapters/services/aws_nova_sonic_adapter.py", + "src/pipecat/adapters/services/bedrock_adapter.py", + "src/pipecat/adapters/services/gemini_adapter.py", + "src/pipecat/adapters/services/grok_realtime_adapter.py", + "src/pipecat/adapters/services/inworld_realtime_adapter.py", + "src/pipecat/adapters/services/open_ai_adapter.py", + "src/pipecat/adapters/services/open_ai_realtime_adapter.py", + "src/pipecat/adapters/services/open_ai_responses_adapter.py", + "src/pipecat/adapters/services/perplexity_adapter.py", + "src/pipecat/audio/dtmf/utils.py", + "src/pipecat/audio/filters/aic_filter.py", + "src/pipecat/audio/filters/krisp_viva_filter.py", + "src/pipecat/audio/filters/rnnoise_filter.py", + "src/pipecat/audio/resamplers/soxr_stream_resampler.py", + "src/pipecat/audio/turn/smart_turn/local_smart_turn_v2.py", + "src/pipecat/audio/turn/smart_turn/local_smart_turn_v3.py", + "src/pipecat/audio/vad/silero.py", + "src/pipecat/processors/aggregators/llm_context.py", + "src/pipecat/processors/aggregators/llm_response_universal.py", + "src/pipecat/processors/frame_processor.py", + "src/pipecat/processors/frameworks/langchain.py", + "src/pipecat/processors/frameworks/rtvi/observer.py", + "src/pipecat/processors/frameworks/rtvi/processor.py", + "src/pipecat/processors/frameworks/strands_agents.py", + "src/pipecat/processors/gstreamer/pipeline_source.py", + "src/pipecat/processors/metrics/frame_processor_metrics.py", + "src/pipecat/services/ai_service.py", + "src/pipecat/services/anthropic/llm.py", + "src/pipecat/services/assemblyai/stt.py", + "src/pipecat/services/asyncai/tts.py", + "src/pipecat/services/aws/agent_core.py", + "src/pipecat/services/aws/llm.py", + "src/pipecat/services/aws/nova_sonic/llm.py", + "src/pipecat/services/aws/sagemaker/bidi_client.py", + "src/pipecat/services/aws/stt.py", + "src/pipecat/services/aws/tts.py", + "src/pipecat/services/aws/utils.py", + "src/pipecat/services/azure/stt.py", + "src/pipecat/services/azure/tts.py", + "src/pipecat/services/camb/tts.py", + "src/pipecat/services/cartesia/stt.py", + "src/pipecat/services/cartesia/tts.py", + "src/pipecat/services/deepgram/flux/base.py", + "src/pipecat/services/deepgram/flux/sagemaker/stt.py", + "src/pipecat/services/deepgram/flux/stt.py", + "src/pipecat/services/deepgram/sagemaker/stt.py", + "src/pipecat/services/deepgram/sagemaker/tts.py", + "src/pipecat/services/deepgram/stt.py", + "src/pipecat/services/deepgram/tts.py", + "src/pipecat/services/elevenlabs/stt.py", + "src/pipecat/services/elevenlabs/tts.py", + "src/pipecat/services/fish/tts.py", + "src/pipecat/services/gladia/stt.py", + "src/pipecat/services/google/gemini_live/llm.py", + "src/pipecat/services/google/gemini_live/vertex/llm.py", + "src/pipecat/services/google/image.py", + "src/pipecat/services/google/llm.py", + "src/pipecat/services/google/stt.py", + "src/pipecat/services/google/tts.py", + "src/pipecat/services/google/vertex/llm.py", + "src/pipecat/services/gradium/stt.py", + "src/pipecat/services/gradium/tts.py", + "src/pipecat/services/groq/tts.py", + "src/pipecat/services/heygen/api_interactive_avatar.py", + "src/pipecat/services/heygen/base_api.py", + "src/pipecat/services/heygen/client.py", + "src/pipecat/services/heygen/video.py", + "src/pipecat/services/hume/tts.py", + "src/pipecat/services/image_service.py", + "src/pipecat/services/inworld/realtime/llm.py", + "src/pipecat/services/inworld/tts.py", + "src/pipecat/services/kokoro/tts.py", + "src/pipecat/services/llm_service.py", + "src/pipecat/services/lmnt/tts.py", + "src/pipecat/services/mcp_service.py", + "src/pipecat/services/mem0/memory.py", + "src/pipecat/services/mistral/llm.py", + "src/pipecat/services/mistral/stt.py", + "src/pipecat/services/mistral/tts.py", + "src/pipecat/services/moondream/vision.py", + "src/pipecat/services/neuphonic/tts.py", + "src/pipecat/services/nvidia/stt.py", + "src/pipecat/services/nvidia/tts.py", + "src/pipecat/services/openai/base_llm.py", + "src/pipecat/services/openai/image.py", + "src/pipecat/services/openai/llm.py", + "src/pipecat/services/openai/realtime/llm.py", + "src/pipecat/services/openai/responses/llm.py", + "src/pipecat/services/openai/stt.py", + "src/pipecat/services/openai/tts.py", + "src/pipecat/services/openrouter/llm.py", + "src/pipecat/services/piper/tts.py", + "src/pipecat/services/resembleai/tts.py", + "src/pipecat/services/rime/tts.py", + "src/pipecat/services/sambanova/llm.py", + "src/pipecat/services/sarvam/llm.py", + "src/pipecat/services/sarvam/stt.py", + "src/pipecat/services/sarvam/tts.py", + "src/pipecat/services/simli/video.py", + "src/pipecat/services/smallest/stt.py", + "src/pipecat/services/smallest/tts.py", + "src/pipecat/services/soniox/stt.py", + "src/pipecat/services/speechmatics/stt.py", + "src/pipecat/services/speechmatics/tts.py", + "src/pipecat/services/stt_service.py", + "src/pipecat/services/tavus/video.py", + "src/pipecat/services/tts_service.py", + "src/pipecat/services/ultravox/llm.py", + "src/pipecat/services/vision_service.py", + "src/pipecat/services/websocket_service.py", + "src/pipecat/services/whisper/stt.py", + "src/pipecat/services/xai/realtime/events.py", + "src/pipecat/services/xai/realtime/llm.py", + "src/pipecat/services/xai/stt.py", + "src/pipecat/services/xai/tts.py", + "src/pipecat/services/xtts/tts.py", + "src/pipecat/transports/base_output.py", + "src/pipecat/transports/daily/transport.py", + "src/pipecat/transports/heygen/transport.py", + "src/pipecat/transports/lemonslice/transport.py", + "src/pipecat/transports/livekit/transport.py", + "src/pipecat/transports/local/tk.py", + "src/pipecat/transports/smallwebrtc/connection.py", + "src/pipecat/transports/smallwebrtc/request_handler.py", + "src/pipecat/transports/smallwebrtc/transport.py", + "src/pipecat/transports/tavus/transport.py", + "src/pipecat/transports/websocket/client.py", + "src/pipecat/transports/websocket/fastapi.py", + "src/pipecat/transports/websocket/server.py", + "src/pipecat/transports/whatsapp/client.py" + ], + "reportMissingImports": false +} diff --git a/scripts/daily/test_tavus_transport.py b/scripts/daily/test_tavus_transport.py index fa8afe835..ffb6b8ee6 100644 --- a/scripts/daily/test_tavus_transport.py +++ b/scripts/daily/test_tavus_transport.py @@ -2,7 +2,14 @@ import asyncio import os import signal -from daily import * +from daily import ( + AudioData, + CallClient, + CustomAudioSource, + CustomAudioTrack, + Daily, + EventHandler, +) from dotenv import load_dotenv from loguru import logger @@ -33,8 +40,8 @@ class DailyProxyApp(EventHandler): def __init__(self, sample_rate: int): super().__init__() self._sample_rate = sample_rate - self._loop = None - self._audio_queue: asyncio.Queue | None = None + self._loop = asyncio.new_event_loop() + self._audio_queue: asyncio.Queue = asyncio.Queue() self._audio_task: asyncio.Task | None = None self._client: CallClient = CallClient(event_handler=self) @@ -52,7 +59,6 @@ class DailyProxyApp(EventHandler): self._loop.call_soon_threadsafe(self._loop.stop) def run(self, meeting_url: str): - self._loop = asyncio.new_event_loop() asyncio.set_event_loop(self._loop) self._create_audio_task() @@ -101,7 +107,6 @@ class DailyProxyApp(EventHandler): def _create_audio_task(self): if not self._audio_task: - self._audio_queue = asyncio.Queue() self._audio_task = self._loop.create_task(self._audio_task_handler()) async def _cancel_audio_task(self): @@ -113,7 +118,6 @@ class DailyProxyApp(EventHandler): except asyncio.CancelledError: pass self._audio_task = None - self._audio_queue = None async def capture_participant_audio(self, participant_id: str): logger.info(f"Capturing participant audio: {participant_id}") @@ -168,7 +172,7 @@ class DailyProxyApp(EventHandler): def main(): Daily.init() - room_url = os.getenv("TAVUS_SAMPLE_ROOM_URL") + room_url = os.environ["TAVUS_SAMPLE_ROOM_URL"] app = DailyProxyApp(sample_rate=24000) app.run(room_url) diff --git a/scripts/evals/eval.py b/scripts/evals/eval.py index cf7ecf257..fdec8238c 100644 --- a/scripts/evals/eval.py +++ b/scripts/evals/eval.py @@ -198,7 +198,7 @@ class EvalRunner: async def run_example_pipeline(script_path: Path, eval_config: EvalConfig): - room_url = os.getenv("DAILY_ROOM_URL") + room_url = os.environ["DAILY_ROOM_URL"] module = load_module_from_path(script_path) @@ -227,7 +227,7 @@ async def run_eval_pipeline( ): logger.info(f"Starting eval bot") - room_url = os.getenv("DAILY_ROOM_URL") + room_url = os.environ["DAILY_ROOM_URL"] transport = DailyTransport( room_url, @@ -243,7 +243,7 @@ async def run_eval_pipeline( # We disable smart formatting because some times if the user says "3 + 2 is # 5" (in audio) this can be converted to "32 is 5". stt = DeepgramSTTService( - api_key=os.getenv("DEEPGRAM_API_KEY"), + api_key=os.environ["DEEPGRAM_API_KEY"], settings=DeepgramSTTService.Settings( language="multi", smart_format=False, @@ -251,7 +251,7 @@ async def run_eval_pipeline( ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), + api_key=os.environ["CARTESIA_API_KEY"], settings=CartesiaTTSService.Settings( voice="97f4b8fb-f2fe-444b-bb9a-c109783a857a", # Nathan ), @@ -375,7 +375,7 @@ async def run_eval_pipeline( @task.event_handler("on_pipeline_finished") async def on_pipeline_finished(task, frame): if isinstance(frame, EndFrame): - await eval_runner.assert_eval(frame.reason) + await eval_runner.assert_eval(bool(frame.reason)) elif isinstance(frame, CancelFrame): await eval_runner.assert_eval(False) diff --git a/scripts/evals/run-eval.py b/scripts/evals/run-eval.py index 72d4902b7..5ccb4f7c6 100644 --- a/scripts/evals/run-eval.py +++ b/scripts/evals/run-eval.py @@ -11,7 +11,7 @@ import sys from pathlib import Path from dotenv import load_dotenv -from eval import EvalRunner +from eval import EvalConfig, EvalRunner from loguru import logger from utils import check_env_variables @@ -33,7 +33,8 @@ async def main(args: argparse.Namespace): runner = EvalRunner(examples_dir=script_path, record_audio=args.audio, log_level=log_level) - await runner.run_eval(script_file, args.prompt, args.eval) + eval_config = EvalConfig(prompt=args.prompt, eval=args.eval) + await runner.run_eval(script_file, eval_config) runner.print_results() diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index ebf5b36c2..861367027 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -108,6 +108,7 @@ TESTS_VOICE = [ ("voice/voice-elevenlabs.py", EVAL_SIMPLE_MATH), ("voice/voice-elevenlabs-http.py", EVAL_SIMPLE_MATH), ("voice/voice-xai.py", EVAL_SIMPLE_MATH), + ("voice/voice-xai-http.py", EVAL_SIMPLE_MATH), ("voice/voice-azure.py", EVAL_SIMPLE_MATH), ("voice/voice-azure-http.py", EVAL_SIMPLE_MATH), ("voice/voice-openai.py", EVAL_SIMPLE_MATH), diff --git a/scripts/evals/utils.py b/scripts/evals/utils.py index 7ce9dfe84..997ef8f73 100644 --- a/scripts/evals/utils.py +++ b/scripts/evals/utils.py @@ -79,6 +79,8 @@ def load_module_from_path(path: str | Path): module_name = path.stem spec = importlib.util.spec_from_file_location(module_name, str(path)) + if spec is None or spec.loader is None: + raise ImportError(f"Could not load module spec from {path}") module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module diff --git a/scripts/fix-ruff.sh b/scripts/fix-ruff-and-typecheck.sh similarity index 83% rename from scripts/fix-ruff.sh rename to scripts/fix-ruff-and-typecheck.sh index 96c6a278b..31cce6dc3 100755 --- a/scripts/fix-ruff.sh +++ b/scripts/fix-ruff-and-typecheck.sh @@ -6,5 +6,9 @@ PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" echo "Running ruff format..." uv run ruff format "$PROJECT_ROOT" + echo "Running ruff check..." uv run ruff check --fix "$PROJECT_ROOT" + +echo "Running pyright check..." +uv run pyright diff --git a/scripts/krisp/test_krisp_viva_turn_audiofile.py b/scripts/krisp/test_krisp_viva_turn_audiofile.py index 6580e8e90..99557c171 100644 --- a/scripts/krisp/test_krisp_viva_turn_audiofile.py +++ b/scripts/krisp/test_krisp_viva_turn_audiofile.py @@ -71,7 +71,7 @@ async def analyze_audio_file( frame_duration_ms: int = 20, chunk_duration_ms: int = 20, verbose: bool = False, - output_file: str = None, + output_file: str | None = None, ) -> None: """Analyze an audio file for turn detection using Krisp VIVA turn analyzer. diff --git a/src/pipecat/audio/krisp_instance.py b/src/pipecat/audio/krisp_instance.py index 5ebfd24cc..afb5848ee 100644 --- a/src/pipecat/audio/krisp_instance.py +++ b/src/pipecat/audio/krisp_instance.py @@ -15,9 +15,11 @@ from loguru import logger try: import krisp_audio except ModuleNotFoundError as e: - logger.error(f"Exception: {e}") - logger.error("In order to use the Krisp instance, you need to install krisp_audio.") - raise Exception(f"Missing module: {e}") + raise ImportError( + "krisp_audio is required for Krisp audio features. " + "Install it to use KrispVivaFilter, KrispVivaVadAnalyzer, " + "KrispVivaTurn, or KrispVivaIPUserTurnStartStrategy." + ) from e # Mapping of sample rates (Hz) to Krisp SDK SamplingRate enums diff --git a/src/pipecat/audio/turn/krisp_viva_turn.py b/src/pipecat/audio/turn/krisp_viva_turn.py index 5235a94be..a9cf6a847 100644 --- a/src/pipecat/audio/turn/krisp_viva_turn.py +++ b/src/pipecat/audio/turn/krisp_viva_turn.py @@ -7,7 +7,9 @@ """Krisp turn analyzer for end-of-turn detection using Krisp VIVA SDK. This module provides a turn analyzer implementation using Krisp's turn detection -(Tt) API to determine when a user has finished speaking in a conversation. +v3 (Tt) API to determine when a user has finished speaking in a conversation. +The Tt API accepts an external VAD flag alongside audio frames, allowing the +model to leverage voice activity information for more accurate turn detection. Note: This analyzer uses a different model than KrispVivaFilter. The model path can be specified via the KRISP_VIVA_TURN_MODEL_PATH environment variable or @@ -33,7 +35,7 @@ try: except ModuleNotFoundError as e: logger.error(f"Exception: {e}") logger.error("In order to use KrispVivaTurn, you need to install krisp_audio.") - raise Exception(f"Missing module: {e}") + raise ImportError(f"Missing module: {e}") from e class KrispTurnParams(BaseTurnParams): @@ -53,8 +55,10 @@ class KrispTurnParams(BaseTurnParams): class KrispVivaTurn(BaseTurnAnalyzer): """Turn analyzer using Krisp VIVA SDK for end-of-turn detection. - Uses Krisp's turn detection (Tt) API to determine when a user has finished - speaking. This analyzer requires a valid Krisp model file to operate. + Uses Krisp's turn detection v3 (Tt) API to determine when a user has + finished speaking. The Tt API receives an external VAD flag with each + audio frame, which the ``is_speech`` parameter of ``append_audio`` + provides. This analyzer requires a valid Krisp model file to operate. """ def __init__( @@ -158,14 +162,14 @@ class KrispVivaTurn(BaseTurnAnalyzer): """Create a turn detection session with the specified sample rate. Args: - sample_rate: Sample rate for the session + sample_rate: Sample rate for the session. Returns: - krisp_audio.TtFloat instance + krisp_audio.TtFloat instance. Raises: - ValueError: If sample rate or frame duration is not supported - RuntimeError: If session creation fails + ValueError: If sample rate or frame duration is not supported. + RuntimeError: If session creation fails. """ try: model_info = krisp_audio.ModelInfo() @@ -306,12 +310,7 @@ class KrispVivaTurn(BaseTurnAnalyzer): # Instead, we wait for the model's probability check below to confirm # end-of-turn based on the threshold. - prob = self._tt_session.process(frame.tolist()) - - # Negative values indicate the model is not ready yet (working with 100ms data) - # Skip processing until we get positive probabilities - if prob < 0: - continue + prob = self._tt_session.process(frame.tolist(), is_speech, False) # Store the probability for external access self._last_probability = prob diff --git a/src/pipecat/extensions/ivr/ivr_navigator.py b/src/pipecat/extensions/ivr/ivr_navigator.py index f6b36655b..37b2469bb 100644 --- a/src/pipecat/extensions/ivr/ivr_navigator.py +++ b/src/pipecat/extensions/ivr/ivr_navigator.py @@ -30,6 +30,7 @@ from pipecat.frames.frames import ( VADParamsUpdateFrame, ) from pipecat.pipeline.pipeline import Pipeline +from pipecat.processors.aggregators.llm_context import LLMContextMessage from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.llm_service import LLMService from pipecat.utils.text.pattern_pair_aggregator import ( @@ -87,7 +88,7 @@ class IVRProcessor(FrameProcessor): self._classifier_prompt = classifier_prompt # Store saved context messages - self._saved_messages: list[dict] = [] + self._saved_messages: list[LLMContextMessage] = [] # XML pattern aggregation self._aggregator = PatternPairAggregator() @@ -97,18 +98,18 @@ class IVRProcessor(FrameProcessor): self._register_event_handler("on_conversation_detected") self._register_event_handler("on_ivr_status_changed") - def update_saved_messages(self, messages: list[dict]) -> None: + def update_saved_messages(self, messages: list[LLMContextMessage]) -> None: """Update the saved context messages. Sets the messages that are saved when switching between conversation and IVR navigation modes. Args: - messages: List of message dictionaries to save. + messages: List of context messages to save. """ self._saved_messages = messages - def _get_conversation_history(self) -> list[dict]: + def _get_conversation_history(self) -> list[LLMContextMessage]: """Get saved context messages without the system message. Returns: @@ -144,7 +145,9 @@ class IVRProcessor(FrameProcessor): await self.push_frame(frame, direction) # Set the classifier prompt and push it upstream - messages = [{"role": "system", "content": self._classifier_prompt}] + messages: list[LLMContextMessage] = [ + {"role": "developer", "content": self._classifier_prompt} + ] llm_update_frame = LLMMessagesUpdateFrame(messages=messages) await self.push_frame(llm_update_frame, FrameDirection.UPSTREAM) @@ -261,7 +264,7 @@ class IVRProcessor(FrameProcessor): logger.debug("IVR detected - switching to IVR navigation mode") # Create new context with IVR system prompt and saved messages - messages = [{"role": "system", "content": self._ivr_prompt}] + messages: list[LLMContextMessage] = [{"role": "developer", "content": self._ivr_prompt}] # Add saved conversation history if available conversation_history = self._get_conversation_history() diff --git a/src/pipecat/extensions/voicemail/voicemail_detector.py b/src/pipecat/extensions/voicemail/voicemail_detector.py index 3ab7f2f7d..d7e9d559b 100644 --- a/src/pipecat/extensions/voicemail/voicemail_detector.py +++ b/src/pipecat/extensions/voicemail/voicemail_detector.py @@ -35,7 +35,7 @@ from pipecat.frames.frames import ( UserStoppedSpeakingFrame, ) from pipecat.pipeline.parallel_pipeline import ParallelPipeline -from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_context import LLMContext, LLMContextMessage from pipecat.processors.aggregators.llm_response_universal import ( LLMContextAggregatorPair, LLMUserAggregatorParams, @@ -617,9 +617,9 @@ VOICEMAIL SYSTEM (respond "VOICEMAIL"): self._validate_prompt(custom_system_prompt) # Set up the LLM context with the classification prompt - self._messages = [ + self._messages: list[LLMContextMessage] = [ { - "role": "system", + "role": "developer", "content": self._prompt, }, ] diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 7fd215caf..24006428d 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -20,7 +20,6 @@ from typing import ( TYPE_CHECKING, Any, Literal, - Optional, ) from pipecat.adapters.schemas.tools_schema import ToolsSchema @@ -559,11 +558,11 @@ class LLMMessagesAppendFrame(DataFrame): current context. Parameters: - messages: List of message dictionaries to append. + messages: List of context messages to append. run_llm: Whether the context update should be sent to the LLM. """ - messages: list[dict] + messages: list[LLMContextMessage] run_llm: bool | None = None @@ -575,11 +574,11 @@ class LLMMessagesUpdateFrame(DataFrame): context LLM messages. Parameters: - messages: List of message dictionaries to replace current context. + messages: List of context messages to replace current context. run_llm: Whether the context update should be sent to the LLM. """ - messages: list[dict] + messages: list[LLMContextMessage] run_llm: bool | None = None diff --git a/src/pipecat/observers/startup_timing_observer.py b/src/pipecat/observers/startup_timing_observer.py index 6c6eb8204..ac63ab10a 100644 --- a/src/pipecat/observers/startup_timing_observer.py +++ b/src/pipecat/observers/startup_timing_observer.py @@ -49,6 +49,15 @@ from pipecat.processors.frame_processor import FrameProcessor _INTERNAL_TYPES = (PipelineSource, BasePipeline) +@dataclass +class _StartFrameInfo: + """Captured once when the first StartFrame arrives at a processor.""" + + frame_id: int + arrival_ns: int + wall_clock: float + + @dataclass class _ArrivalInfo: """Internal record of when a StartFrame arrived at a processor.""" @@ -175,8 +184,8 @@ class StartupTimingObserver(BaseObserver): # Collected timings in pipeline order. self._timings: list[ProcessorStartupTiming] = [] - # Lock onto the first StartFrame we see (by frame ID). - self._start_frame_id: str | None = None + # Captured once when the first StartFrame arrives. + self._start_frame: _StartFrameInfo | None = None # Whether we've already emitted the startup timing report. self._startup_timing_reported = False @@ -184,15 +193,9 @@ class StartupTimingObserver(BaseObserver): # Whether we've already measured transport timing. self._transport_timing_reported = False - # Timestamp (ns) when we first see a StartFrame arrive at a processor. - self._start_frame_arrival_ns: int | None = None - # Bot connected timing (stored for inclusion in the transport report). self._bot_connected_secs: float | None = None - # Wall clock time when the StartFrame was first seen. - self._start_wall_clock: float | None = None - self._register_event_handler("on_startup_timing_report") self._register_event_handler("on_transport_timing_report") @@ -233,11 +236,13 @@ class StartupTimingObserver(BaseObserver): return # Lock onto the first StartFrame. - if self._start_frame_id is None: - self._start_frame_id = data.frame.id - self._start_frame_arrival_ns = data.timestamp - self._start_wall_clock = time.time() - elif data.frame.id != self._start_frame_id: + if self._start_frame is None: + self._start_frame = _StartFrameInfo( + frame_id=data.frame.id, + arrival_ns=data.timestamp, + wall_clock=time.time(), + ) + elif data.frame.id != self._start_frame.frame_id: return if self._should_track(data.processor): @@ -268,16 +273,16 @@ class StartupTimingObserver(BaseObserver): if not isinstance(data.frame, StartFrame): return - if self._start_frame_id is not None and data.frame.id != self._start_frame_id: + if self._start_frame is not None and data.frame.id != self._start_frame.frame_id: return arrival = self._arrivals.pop(data.source.id, None) - if arrival is None: + if arrival is None or self._start_frame is None: return duration_ns = data.timestamp - arrival.arrival_ts_ns duration_secs = duration_ns / 1e9 - start_offset_secs = (arrival.arrival_ts_ns - self._start_frame_arrival_ns) / 1e9 + start_offset_secs = (arrival.arrival_ts_ns - self._start_frame.arrival_ns) / 1e9 self._timings.append( ProcessorStartupTiming( @@ -289,22 +294,22 @@ class StartupTimingObserver(BaseObserver): def _handle_bot_connected(self, data: FramePushed): """Record bot connected timing on first BotConnectedFrame.""" - if self._bot_connected_secs is not None or self._start_frame_arrival_ns is None: + if self._bot_connected_secs is not None or self._start_frame is None: return - delta_ns = data.timestamp - self._start_frame_arrival_ns + delta_ns = data.timestamp - self._start_frame.arrival_ns self._bot_connected_secs = delta_ns / 1e9 async def _handle_client_connected(self, data: FramePushed): """Emit transport timing report on first ClientConnectedFrame.""" - if self._transport_timing_reported or self._start_frame_arrival_ns is None: + if self._transport_timing_reported or self._start_frame is None: return self._transport_timing_reported = True - delta_ns = data.timestamp - self._start_frame_arrival_ns + delta_ns = data.timestamp - self._start_frame.arrival_ns client_connected_secs = delta_ns / 1e9 report = TransportTimingReport( - start_time=self._start_wall_clock or 0.0, + start_time=self._start_frame.wall_clock, bot_connected_secs=self._bot_connected_secs, client_connected_secs=client_connected_secs, ) @@ -319,7 +324,7 @@ class StartupTimingObserver(BaseObserver): total = sum(t.duration_secs for t in self._timings) report = StartupTimingReport( - start_time=self._start_wall_clock or 0.0, + start_time=self._start_frame.wall_clock if self._start_frame else 0.0, total_duration_secs=total, processor_timings=self._timings, ) diff --git a/src/pipecat/pipeline/llm_switcher.py b/src/pipecat/pipeline/llm_switcher.py index 71a7c7974..9c89641e3 100644 --- a/src/pipecat/pipeline/llm_switcher.py +++ b/src/pipecat/pipeline/llm_switcher.py @@ -6,7 +6,7 @@ """LLM switcher for switching between different LLMs at runtime, with different switching strategies.""" -from typing import Any +from typing import Any, cast from pipecat.adapters.schemas.direct_function import DirectFunction from pipecat.pipeline.service_switcher import ( @@ -15,6 +15,7 @@ from pipecat.pipeline.service_switcher import ( StrategyType, ) from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.frame_processor import FrameProcessor from pipecat.services.llm_service import LLMService @@ -38,7 +39,7 @@ class LLMSwitcher(ServiceSwitcher[StrategyType]): strategy_type: The strategy class to use for switching between LLMs. Defaults to ``ServiceSwitcherStrategyManual``. """ - super().__init__(llms, strategy_type) + super().__init__(cast(list[FrameProcessor], llms), strategy_type) @property def llms(self) -> list[LLMService]: @@ -47,7 +48,7 @@ class LLMSwitcher(ServiceSwitcher[StrategyType]): Returns: List of LLM services managed by this switcher. """ - return self.services + return cast(list[LLMService], self.services) @property def active_llm(self) -> LLMService: @@ -56,7 +57,7 @@ class LLMSwitcher(ServiceSwitcher[StrategyType]): Returns: The currently active LLM service, or None if no LLM is active. """ - return self.strategy.active_service + return cast(LLMService, self.strategy.active_service) async def run_inference(self, context: LLMContext, **kwargs) -> str | None: """Run a one-shot, out-of-band (i.e. out-of-pipeline) inference with the given LLM context, using the currently active LLM. diff --git a/src/pipecat/pipeline/pipeline.py b/src/pipecat/pipeline/pipeline.py index 325cedb82..d7999b4fe 100644 --- a/src/pipecat/pipeline/pipeline.py +++ b/src/pipecat/pipeline/pipeline.py @@ -11,7 +11,7 @@ in sequence and manages frame flow between them, along with helper classes for pipeline source and sink operations. """ -from collections.abc import Callable, Coroutine +from collections.abc import Callable, Coroutine, Sequence from pipecat.frames.frames import Frame from pipecat.pipeline.base_pipeline import BasePipeline @@ -98,7 +98,7 @@ class Pipeline(BasePipeline): def __init__( self, - processors: list[FrameProcessor], + processors: Sequence[FrameProcessor], *, source: FrameProcessor | None = None, sink: FrameProcessor | None = None, @@ -106,7 +106,7 @@ class Pipeline(BasePipeline): """Initialize the pipeline with a list of processors. Args: - processors: List of frame processors to connect in sequence. + processors: Sequence of frame processors to connect in sequence. source: An optional pipeline source processor. sink: An optional pipeline sink processor. """ @@ -116,7 +116,7 @@ class Pipeline(BasePipeline): # downstream outside of the pipeline. self._source = source or PipelineSource(self.push_frame, name=f"{self}::Source") self._sink = sink or PipelineSink(self.push_frame, name=f"{self}::Sink") - self._processors: list[FrameProcessor] = [self._source] + processors + [self._sink] + self._processors: list[FrameProcessor] = [self._source, *processors, self._sink] self._link_processors() diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index 394b3d2e2..08e08ea00 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -742,7 +742,7 @@ class PipelineTask(BasePipelineTask): await self._observer.cleanup() # End conversation tracing if it's active - this will also close any active turn span - if self._enable_tracing and hasattr(self, "_turn_trace_observer"): + if self._enable_tracing and self._turn_trace_observer: self._turn_trace_observer.end_conversation_tracing() # Cleanup pipeline processors. diff --git a/src/pipecat/pipeline/task_observer.py b/src/pipecat/pipeline/task_observer.py index c6603c1d8..5b29850a6 100644 --- a/src/pipecat/pipeline/task_observer.py +++ b/src/pipecat/pipeline/task_observer.py @@ -173,6 +173,8 @@ class TaskObserver(BaseObserver): return proxies async def _send_to_proxy(self, data: Any): + if not self._proxies: + return for proxy in self._proxies.values(): await proxy.queue.put(data) diff --git a/src/pipecat/processors/metrics/sentry.py b/src/pipecat/processors/metrics/sentry.py index b043b9058..0484995db 100644 --- a/src/pipecat/processors/metrics/sentry.py +++ b/src/pipecat/processors/metrics/sentry.py @@ -97,13 +97,20 @@ class SentryMetrics(FrameProcessorMetrics): Args: end_time: Optional end timestamp override. + + Returns: + MetricsFrame produced by the base class, or None if not measuring. + Returning the frame is required so ``FrameProcessor.stop_ttfb_metrics`` + can push it downstream to observers. """ - await super().stop_ttfb_metrics(end_time=end_time) + frame = await super().stop_ttfb_metrics(end_time=end_time) if self._sentry_available and self._ttfb_metrics_tx: await self._sentry_queue.put(self._ttfb_metrics_tx) self._ttfb_metrics_tx = None + return frame + async def start_processing_metrics(self, *, start_time: float | None = None): """Start tracking frame processing metrics. @@ -126,13 +133,20 @@ class SentryMetrics(FrameProcessorMetrics): Args: end_time: Optional end timestamp override. + + Returns: + MetricsFrame produced by the base class, or None if not measuring. + Returning the frame is required so ``FrameProcessor.stop_processing_metrics`` + can push it downstream to observers. """ - await super().stop_processing_metrics(end_time=end_time) + frame = await super().stop_processing_metrics(end_time=end_time) if self._sentry_available and self._processing_metrics_tx: await self._sentry_queue.put(self._processing_metrics_tx) self._processing_metrics_tx = None + return frame + async def _sentry_task_handler(self): """Background task handler for completing Sentry transactions.""" running = True diff --git a/src/pipecat/runner/daily.py b/src/pipecat/runner/daily.py index bc80f0641..168b2353d 100644 --- a/src/pipecat/runner/daily.py +++ b/src/pipecat/runner/daily.py @@ -228,7 +228,7 @@ async def configure( room_properties.enable_dialout = True # Add SIP configuration if enabled - if sip_enabled: + if sip_enabled and sip_caller_phone: sip_params = DailyRoomSipParams( display_name=sip_caller_phone, video=sip_enable_video, diff --git a/src/pipecat/runner/run.py b/src/pipecat/runner/run.py index c6d43fbbd..a53dea519 100644 --- a/src/pipecat/runner/run.py +++ b/src/pipecat/runner/run.py @@ -156,6 +156,8 @@ def _get_bot_module(): spec = importlib.util.spec_from_file_location( module_name, os.path.join(cwd, filename) ) + if spec is None or spec.loader is None: + continue module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) @@ -386,29 +388,27 @@ def _add_lifespan_to_app(app: FastAPI, new_lifespan): def _setup_whatsapp_routes(app: FastAPI, args: argparse.Namespace): """Set up WhatsApp-specific routes.""" - WHATSAPP_APP_SECRET = os.getenv("WHATSAPP_APP_SECRET") - WHATSAPP_PHONE_NUMBER_ID = os.getenv("WHATSAPP_PHONE_NUMBER_ID") - WHATSAPP_TOKEN = os.getenv("WHATSAPP_TOKEN") - WHATSAPP_WEBHOOK_VERIFICATION_TOKEN = os.getenv("WHATSAPP_WEBHOOK_VERIFICATION_TOKEN") - - if not all( - [ - WHATSAPP_APP_SECRET, - WHATSAPP_PHONE_NUMBER_ID, - WHATSAPP_TOKEN, - WHATSAPP_WEBHOOK_VERIFICATION_TOKEN, - ] - ): + required_vars = [ + "WHATSAPP_APP_SECRET", + "WHATSAPP_PHONE_NUMBER_ID", + "WHATSAPP_TOKEN", + "WHATSAPP_WEBHOOK_VERIFICATION_TOKEN", + ] + missing = [v for v in required_vars if not os.getenv(v)] + if missing: + missing_list = "\n ".join(missing) logger.error( - """Missing required environment variables for WhatsApp transport: - WHATSAPP_APP_SECRET - WHATSAPP_PHONE_NUMBER_ID - WHATSAPP_TOKEN - WHATSAPP_WEBHOOK_VERIFICATION_TOKEN + f"""Missing required environment variables for WhatsApp transport: + {missing_list} """ ) return + WHATSAPP_APP_SECRET = os.environ["WHATSAPP_APP_SECRET"] + WHATSAPP_PHONE_NUMBER_ID = os.environ["WHATSAPP_PHONE_NUMBER_ID"] + WHATSAPP_TOKEN = os.environ["WHATSAPP_TOKEN"] + WHATSAPP_WEBHOOK_VERIFICATION_TOKEN = os.environ["WHATSAPP_WEBHOOK_VERIFICATION_TOKEN"] + try: from pipecat.transports.smallwebrtc.connection import SmallWebRTCConnection from pipecat.transports.whatsapp.api import WhatsAppWebhookRequest diff --git a/src/pipecat/runner/types.py b/src/pipecat/runner/types.py index 055824a22..b009b3350 100644 --- a/src/pipecat/runner/types.py +++ b/src/pipecat/runner/types.py @@ -122,4 +122,4 @@ class LiveKitRunnerArguments(RunnerArguments): room_name: str url: str - token: str | None = None + token: str diff --git a/src/pipecat/runner/utils.py b/src/pipecat/runner/utils.py index 7a4b3034c..84316a743 100644 --- a/src/pipecat/runner/utils.py +++ b/src/pipecat/runner/utils.py @@ -416,9 +416,9 @@ def _get_transport_params(transport_key: str, transport_params: dict[str, Callab async def _create_telephony_transport( websocket: WebSocket, - params: Any | None = None, - transport_type: str = None, - call_data: dict = None, + params: Any, + transport_type: str, + call_data: dict, ) -> BaseTransport: """Create a telephony transport with pre-parsed WebSocket data. @@ -433,12 +433,6 @@ async def _create_telephony_transport( """ from pipecat.transports.websocket.fastapi import FastAPIWebsocketTransport - if params is None: - raise ValueError( - "FastAPIWebsocketParams must be provided. " - "The serializer and add_wav_header will be set automatically." - ) - # Always set add_wav_header to False for telephony params.add_wav_header = False diff --git a/src/pipecat/serializers/exotel.py b/src/pipecat/serializers/exotel.py index ff2510f57..90d28270a 100644 --- a/src/pipecat/serializers/exotel.py +++ b/src/pipecat/serializers/exotel.py @@ -59,7 +59,9 @@ class ExotelFrameSerializer(FrameSerializer): call_sid: The associated Exotel Call SID (optional, not used in this implementation). params: Configuration parameters. """ - super().__init__(params or ExotelFrameSerializer.InputParams()) + params = params or ExotelFrameSerializer.InputParams() + super().__init__(params) + self._params: ExotelFrameSerializer.InputParams = params self._stream_sid = stream_sid self._call_sid = call_sid diff --git a/src/pipecat/serializers/genesys.py b/src/pipecat/serializers/genesys.py index e52abc6aa..9fca1f89a 100644 --- a/src/pipecat/serializers/genesys.py +++ b/src/pipecat/serializers/genesys.py @@ -166,7 +166,9 @@ class GenesysAudioHookSerializer(FrameSerializer): params: Configuration parameters. **kwargs: Additional arguments passed to BaseObject (e.g., name). """ - super().__init__(params or GenesysAudioHookSerializer.InputParams(), **kwargs) + params = params or GenesysAudioHookSerializer.InputParams() + super().__init__(params, **kwargs) + self._params: GenesysAudioHookSerializer.InputParams = params self._genesys_sample_rate = self._params.genesys_sample_rate self._sample_rate = 0 # Pipeline input rate, set in setup() diff --git a/src/pipecat/serializers/plivo.py b/src/pipecat/serializers/plivo.py index e86d18b8c..0df2d6753 100644 --- a/src/pipecat/serializers/plivo.py +++ b/src/pipecat/serializers/plivo.py @@ -8,6 +8,7 @@ import base64 import json +from typing import cast from loguru import logger @@ -71,7 +72,24 @@ class PlivoFrameSerializer(FrameSerializer): auth_token: Plivo auth token (required for auto hang-up). params: Configuration parameters. """ - super().__init__(params or PlivoFrameSerializer.InputParams()) + params = params or PlivoFrameSerializer.InputParams() + super().__init__(params) + self._params: PlivoFrameSerializer.InputParams = params + + # Validate hangup-related parameters if auto_hang_up is enabled + if self._params.auto_hang_up: + missing_credentials = [] + if not call_id: + missing_credentials.append("call_id") + if not auth_id: + missing_credentials.append("auth_id") + if not auth_token: + missing_credentials.append("auth_token") + + if missing_credentials: + raise ValueError( + f"auto_hang_up is enabled but missing required parameters: {', '.join(missing_credentials)}" + ) self._stream_id = stream_id self._call_id = call_id @@ -152,23 +170,11 @@ class PlivoFrameSerializer(FrameSerializer): try: import aiohttp - auth_id = self._auth_id - auth_token = self._auth_token - call_id = self._call_id - - if not call_id or not auth_id or not auth_token: - missing = [] - if not call_id: - missing.append("call_id") - if not auth_id: - missing.append("auth_id") - if not auth_token: - missing.append("auth_token") - - logger.warning( - f"Cannot hang up Plivo call: missing required parameters: {', '.join(missing)}" - ) - return + # __init__ guarantees these are non-None whenever auto_hang_up is True, + # which is the only path that reaches this method. + auth_id = cast(str, self._auth_id) + auth_token = cast(str, self._auth_token) + call_id = cast(str, self._call_id) # Plivo API endpoint for hanging up calls endpoint = f"https://api.plivo.com/v1/Account/{auth_id}/Call/{call_id}/" diff --git a/src/pipecat/serializers/protobuf.py b/src/pipecat/serializers/protobuf.py index 78d20fa24..44c660042 100644 --- a/src/pipecat/serializers/protobuf.py +++ b/src/pipecat/serializers/protobuf.py @@ -83,23 +83,24 @@ class ProtobufFrameSerializer(FrameSerializer): Serialized frame as bytes, or None if frame type is not serializable. """ # Wrapping this messages as a JSONFrame to send + serializable: Frame | MessageFrame = frame if isinstance(frame, (OutputTransportMessageFrame, OutputTransportMessageUrgentFrame)): if self.should_ignore_frame(frame): return None - frame = MessageFrame( + serializable = MessageFrame( data=json.dumps(frame.message), ) - proto_frame = frame_protos.Frame() - if type(frame) not in self.SERIALIZABLE_TYPES: - logger.warning(f"Frame type {type(frame)} is not serializable") + proto_frame = frame_protos.Frame() # type: ignore[attr-defined] + if type(serializable) not in self.SERIALIZABLE_TYPES: + logger.warning(f"Frame type {type(serializable)} is not serializable") return None # ignoring linter errors; we check that type(frame) is in this dict above - proto_optional_name = self.SERIALIZABLE_TYPES[type(frame)] # type: ignore + proto_optional_name = self.SERIALIZABLE_TYPES[type(serializable)] # type: ignore proto_attr = getattr(proto_frame, proto_optional_name) - for field in dataclasses.fields(frame): # type: ignore - value = getattr(frame, field.name) + for field in dataclasses.fields(serializable): # type: ignore + value = getattr(serializable, field.name) if value and hasattr(proto_attr, field.name): setattr(proto_attr, field.name, value) @@ -114,7 +115,7 @@ class ProtobufFrameSerializer(FrameSerializer): Returns: Deserialized frame instance, or None if deserialization fails. """ - proto = frame_protos.Frame.FromString(data) + proto = frame_protos.Frame.FromString(data) # type: ignore[attr-defined] which = proto.WhichOneof("frame") if which not in self.DESERIALIZABLE_FIELDS: logger.error("Unable to deserialize a valid frame") diff --git a/src/pipecat/serializers/telnyx.py b/src/pipecat/serializers/telnyx.py index 0d74664ab..4d769cada 100644 --- a/src/pipecat/serializers/telnyx.py +++ b/src/pipecat/serializers/telnyx.py @@ -8,10 +8,10 @@ import base64 import json +from typing import cast import aiohttp from loguru import logger -from pydantic import BaseModel from pipecat.audio.dtmf.types import KeypadEntry from pipecat.audio.utils import ( @@ -46,7 +46,7 @@ class TelnyxFrameSerializer(FrameSerializer): credentials to be provided. """ - class InputParams(BaseModel): + class InputParams(FrameSerializer.InputParams): """Configuration parameters for TelnyxFrameSerializer. Parameters: @@ -82,10 +82,26 @@ class TelnyxFrameSerializer(FrameSerializer): api_key: Your Telnyx API key (required for auto hang-up). params: Configuration parameters. """ + params = params or TelnyxFrameSerializer.InputParams() + super().__init__(params) + self._params: TelnyxFrameSerializer.InputParams = params + + # Validate hangup-related parameters if auto_hang_up is enabled + if self._params.auto_hang_up: + missing_credentials = [] + if not call_control_id: + missing_credentials.append("call_control_id") + if not api_key: + missing_credentials.append("api_key") + + if missing_credentials: + raise ValueError( + f"auto_hang_up is enabled but missing required parameters: {', '.join(missing_credentials)}" + ) + self._stream_id = stream_id self._call_control_id = call_control_id self._api_key = api_key - self._params = params or TelnyxFrameSerializer.InputParams() self._params.outbound_encoding = outbound_encoding self._params.inbound_encoding = inbound_encoding @@ -163,14 +179,10 @@ class TelnyxFrameSerializer(FrameSerializer): async def _hang_up_call(self): """Hang up the Telnyx call using Telnyx's REST API.""" try: - call_control_id = self._call_control_id - api_key = self._api_key - - if not call_control_id or not api_key: - logger.warning( - "Cannot hang up Telnyx call: call_control_id and api_key must be provided" - ) - return + # __init__ guarantees these are non-None whenever auto_hang_up is True, + # which is the only path that reaches this method. + call_control_id = cast(str, self._call_control_id) + api_key = cast(str, self._api_key) # Telnyx API endpoint for hanging up a call endpoint = f"https://api.telnyx.com/v2/calls/{call_control_id}/actions/hangup" diff --git a/src/pipecat/serializers/twilio.py b/src/pipecat/serializers/twilio.py index 857610b4e..d93543c8e 100644 --- a/src/pipecat/serializers/twilio.py +++ b/src/pipecat/serializers/twilio.py @@ -8,6 +8,7 @@ import base64 import json +from typing import cast from loguru import logger @@ -75,7 +76,9 @@ class TwilioFrameSerializer(FrameSerializer): edge: Twilio edge location (e.g., "sydney", "dublin"). Must be specified with region. params: Configuration parameters. """ - super().__init__(params or TwilioFrameSerializer.InputParams()) + params = params or TwilioFrameSerializer.InputParams() + super().__init__(params) + self._params: TwilioFrameSerializer.InputParams = params # Validate hangup-related parameters if auto_hang_up is enabled if self._params.auto_hang_up: @@ -178,9 +181,11 @@ class TwilioFrameSerializer(FrameSerializer): try: import aiohttp - account_sid = self._account_sid - auth_token = self._auth_token - call_sid = self._call_sid + # __init__ guarantees these are non-None whenever auto_hang_up is True, + # which is the only path that reaches this method. + account_sid = cast(str, self._account_sid) + auth_token = cast(str, self._auth_token) + call_sid = cast(str, self._call_sid) region = self._region edge = self._edge diff --git a/src/pipecat/serializers/vonage.py b/src/pipecat/serializers/vonage.py index d778cf62c..7d82a717c 100644 --- a/src/pipecat/serializers/vonage.py +++ b/src/pipecat/serializers/vonage.py @@ -54,7 +54,9 @@ class VonageFrameSerializer(FrameSerializer): Args: params: Configuration parameters. """ - super().__init__(params or VonageFrameSerializer.InputParams()) + params = params or VonageFrameSerializer.InputParams() + super().__init__(params) + self._params: VonageFrameSerializer.InputParams = params self._vonage_sample_rate = self._params.vonage_sample_rate self._sample_rate = 0 # Pipeline input rate diff --git a/src/pipecat/services/deepgram/flux/base.py b/src/pipecat/services/deepgram/flux/base.py index baefbd060..c54cdda38 100644 --- a/src/pipecat/services/deepgram/flux/base.py +++ b/src/pipecat/services/deepgram/flux/base.py @@ -27,11 +27,59 @@ from pipecat.frames.frames import ( ) from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven from pipecat.services.stt_service import STTService -from pipecat.transcriptions.language import Language +from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.time import time_now_iso8601 from pipecat.utils.tracing.service_decorators import traced_stt +def language_to_deepgram_flux_language(language: Language) -> str | None: + """Convert a Pipecat Language to a Deepgram Flux language code. + + Only honored by the ``flux-general-multi`` model. Locale variants + (e.g. ``Language.EN_GB``) fall back to the base code. + """ + LANGUAGE_MAP = { + Language.DE: "de", + Language.EN: "en", + Language.ES: "es", + Language.FR: "fr", + Language.HI: "hi", + Language.IT: "it", + Language.JA: "ja", + Language.NL: "nl", + Language.PT: "pt", + Language.RU: "ru", + } + return resolve_language(language, LANGUAGE_MAP, use_base_code=True) + + +def _prepare_language_hints(hints: list[Language] | None) -> list[str]: + """Convert a list of Pipecat Languages to Deepgram Flux codes. + + Drops entries that can't be mapped and deduplicates while preserving order. + """ + if not hints: + return [] + seen: set[str] = set() + prepared: list[str] = [] + for hint in hints: + code = language_to_deepgram_flux_language(hint) + if code is None or code in seen: + continue + seen.add(code) + prepared.append(code) + return prepared + + +def _code_to_pipecat_language(code: str) -> Language | None: + """Convert a Deepgram-returned language code to a Pipecat Language.""" + try: + return Language(code) + except ValueError: + logger.debug(f"Unmapped Deepgram Flux detected language code: {code}") + return None + + class FluxMessageType(StrEnum): """Deepgram Flux WebSocket message types. @@ -73,6 +121,10 @@ class DeepgramFluxSTTSettings(STTSettings): confidence (default 5000). keyterm: Keyterms to boost recognition accuracy for specialized terminology. min_confidence: Minimum confidence required to create a TranscriptionFrame. + language_hints: Languages to bias transcription toward. Only honored by the + ``flux-general-multi`` model. An empty list clears any active hints; + ``None``/``NOT_GIVEN`` means no hints (auto-detect). Can be updated + mid-stream via ``STTUpdateSettingsFrame``. """ eager_eot_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) @@ -80,6 +132,7 @@ class DeepgramFluxSTTSettings(STTSettings): eot_timeout_ms: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) keyterm: list | _NotGiven = field(default_factory=lambda: NOT_GIVEN) min_confidence: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + language_hints: list[Language] | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) class DeepgramFluxSTTBase(STTService): @@ -93,7 +146,14 @@ class DeepgramFluxSTTBase(STTService): Settings = DeepgramFluxSTTSettings _settings: Settings - _CONFIGURE_FIELDS = {"keyterm", "eot_threshold", "eager_eot_threshold", "eot_timeout_ms"} + _CONFIGURE_FIELDS = { + "keyterm", + "eot_threshold", + "eager_eot_threshold", + "eot_timeout_ms", + "language_hints", + } + _MULTILINGUAL_MODEL = "flux-general-multi" def __init__( self, @@ -200,6 +260,18 @@ class DeepgramFluxSTTBase(STTService): for tag_value in self._tag: params.append(urlencode({"tag": tag_value})) + # Add language_hint parameters (only valid on flux-general-multi) + hints = self._settings.language_hints + if hints and not isinstance(hints, _NotGiven): + if self._settings.model == self._MULTILINGUAL_MODEL: + for code in _prepare_language_hints(hints): + params.append(urlencode({"language_hint": code})) + else: + logger.warning( + f"language_hints only supported on {self._MULTILINGUAL_MODEL}; " + f"ignoring hints for model {self._settings.model!r}" + ) + return "&".join(params) async def _send_silence(self, duration_secs: float = 0.5): @@ -266,6 +338,21 @@ class DeepgramFluxSTTBase(STTService): if thresholds: message["thresholds"] = thresholds + if "language_hints" in fields: + if self._settings.model != self._MULTILINGUAL_MODEL: + logger.warning( + f"language_hints only supported on {self._MULTILINGUAL_MODEL}; " + f"skipping Configure update for model {self._settings.model!r}" + ) + else: + hints = self._settings.language_hints + # Empty list clears hints; NOT_GIVEN/None also treated as clear + # since we only reach this branch when the user set the field. + if hints is None or isinstance(hints, _NotGiven): + message["language_hints"] = [] + else: + message["language_hints"] = _prepare_language_hints(hints) + logger.debug(f"{self}: sending Configure message: {message}") await self._transport_send_json(message) @@ -281,8 +368,9 @@ class DeepgramFluxSTTBase(STTService): """Apply a settings delta. Configure-able fields (keyterm, eot_threshold, eager_eot_threshold, - eot_timeout_ms) are sent to Deepgram via a Configure message. - Other fields are stored but cannot be applied to the active connection. + eot_timeout_ms, language_hints) are sent to Deepgram via a Configure + message. Other fields are stored but cannot be applied to the active + connection. """ changed = await super()._update_settings(delta) @@ -520,6 +608,20 @@ class DeepgramFluxSTTBase(STTService): return None return sum(confidences) / len(confidences) + def _primary_detected_language(self, data: dict[str, Any]) -> Language | None: + """Extract the primary detected language from a TurnInfo payload. + + On ``flux-general-multi`` the language is read from TurnInfo's + ``languages`` field. On ``flux-general-en`` the field is absent, so we + fall back to ``Language.EN`` to match the model's fixed language. + """ + codes = data.get("languages") or [] + if codes: + return _code_to_pipecat_language(codes[0]) + if self._settings.model == "flux-general-en": + return Language.EN + return None + async def _handle_end_of_turn(self, transcript: str, data: dict[str, Any]): """Handle EndOfTurn events from Deepgram Flux. @@ -543,6 +645,7 @@ class DeepgramFluxSTTBase(STTService): # Compute the average confidence average_confidence = self._calculate_average_confidence(data) + detected_language = self._primary_detected_language(data) if not self._settings.min_confidence or average_confidence > self._settings.min_confidence: # EndOfTurn means Flux has determined the turn is complete, @@ -552,7 +655,7 @@ class DeepgramFluxSTTBase(STTService): transcript, self._user_id, time_now_iso8601(), - self._settings.language, + detected_language, result=data, finalized=True, ) @@ -562,7 +665,7 @@ class DeepgramFluxSTTBase(STTService): f"Transcription confidence below min_confidence threshold: {average_confidence}" ) - await self._handle_transcription(transcript, True, self._settings.language) + await self._handle_transcription(transcript, True, detected_language) await self.stop_processing_metrics() await self.broadcast_frame(UserStoppedSpeakingFrame) await self._call_event_handler("on_end_of_turn", transcript) @@ -606,7 +709,7 @@ class DeepgramFluxSTTBase(STTService): transcript, self._user_id, time_now_iso8601(), - self._settings.language, + self._primary_detected_language(data), result=data, ) ) diff --git a/src/pipecat/services/deepgram/flux/sagemaker/stt.py b/src/pipecat/services/deepgram/flux/sagemaker/stt.py index da61b169a..94738fc9c 100644 --- a/src/pipecat/services/deepgram/flux/sagemaker/stt.py +++ b/src/pipecat/services/deepgram/flux/sagemaker/stt.py @@ -23,7 +23,6 @@ from pipecat.services.deepgram.flux.base import ( DeepgramFluxSTTBase, DeepgramFluxSTTSettings, ) -from pipecat.transcriptions.language import Language @dataclass @@ -112,12 +111,13 @@ class DeepgramFluxSageMakerSTTService(DeepgramFluxSTTBase): # Initialize default settings default_settings = self.Settings( model="flux-general-en", - language=Language.EN, + language=None, eager_eot_threshold=None, eot_threshold=None, eot_timeout_ms=None, keyterm=[], min_confidence=None, + language_hints=None, ) # Apply settings delta diff --git a/src/pipecat/services/deepgram/flux/stt.py b/src/pipecat/services/deepgram/flux/stt.py index 5b0b16472..6874b1b69 100644 --- a/src/pipecat/services/deepgram/flux/stt.py +++ b/src/pipecat/services/deepgram/flux/stt.py @@ -24,7 +24,6 @@ from pipecat.services.deepgram.flux.base import ( FluxMessageType, ) from pipecat.services.websocket_service import WebsocketService -from pipecat.transcriptions.language import Language try: from websockets.asyncio.client import connect as websocket_connect @@ -50,6 +49,12 @@ class DeepgramFluxSTTService(DeepgramFluxSTTBase, WebsocketService): Supports configurable models, VAD events, and various audio processing options including advanced turn detection and EagerEndOfTurn events for improved conversational AI performance. + For multilingual use, set ``model="flux-general-multi"`` and pass + ``language_hints`` to bias detection toward specific languages. Hints can + be updated mid-stream via ``STTUpdateSettingsFrame`` (e.g. to implement a + detect-then-lock flow). ``TranscriptionFrame.language`` reflects whichever + language Flux detected for each turn. + Event handlers available (in addition to base events): - on_start_of_turn(service, transcript): Deepgram detected start of speech @@ -156,6 +161,16 @@ class DeepgramFluxSTTService(DeepgramFluxSTTBase, WebsocketService): tag=["production", "voice-agent"], ), ) + + Multilingual usage with language hints:: + + stt = DeepgramFluxSTTService( + api_key="your-api-key", + settings=DeepgramFluxSTTService.Settings( + model="flux-general-multi", + language_hints=[Language.EN, Language.ES], + ), + ) """ # Note: For DeepgramFluxSTTService, differently from other processes, we need to create # the _receive_task inside _connect_websocket, because the websocket should only be @@ -171,12 +186,13 @@ class DeepgramFluxSTTService(DeepgramFluxSTTBase, WebsocketService): # 1. Initialize default_settings with hardcoded defaults default_settings = self.Settings( model="flux-general-en", - language=Language.EN, + language=None, eager_eot_threshold=None, eot_threshold=None, eot_timeout_ms=None, keyterm=[], min_confidence=None, + language_hints=None, ) # 2. Apply direct init arg overrides (deprecated) diff --git a/src/pipecat/services/deepgram/stt.py b/src/pipecat/services/deepgram/stt.py index 66b1d70da..c9ebd49c8 100644 --- a/src/pipecat/services/deepgram/stt.py +++ b/src/pipecat/services/deepgram/stt.py @@ -621,6 +621,7 @@ class DeepgramSTTService(STTService): """ while True: connect_kwargs = self._build_connect_kwargs() + keepalive_task = None try: async with self._client.listen.v1.connect(**connect_kwargs) as connection: self._connection = connection @@ -639,7 +640,8 @@ class DeepgramSTTService(STTService): finally: self._connection_ready.clear() self._connection = None - await self.cancel_task(keepalive_task) + if keepalive_task: + await self.cancel_task(keepalive_task) async def _keepalive_handler(self): """Periodically send KeepAlive frames to prevent server-side timeout. diff --git a/src/pipecat/services/elevenlabs/tts.py b/src/pipecat/services/elevenlabs/tts.py index 02e6383ff..f72864ef4 100644 --- a/src/pipecat/services/elevenlabs/tts.py +++ b/src/pipecat/services/elevenlabs/tts.py @@ -245,6 +245,35 @@ class ElevenLabsHttpTTSSettings(TTSSettings): ) +def _strip_leading_space( + alignment: Mapping[str, Any], keys: tuple[str, str, str] +) -> Mapping[str, Any]: + """Return alignment with a prepended space char removed, if present. + + Normalized alignment chunks from ElevenLabs begin with a leading space that + marks the prosody/chunk boundary. Left in place, it would prematurely + terminate a partial word carried over from the previous chunk. Stripping it + is lossless for timing: the dropped space's duration is still reflected in + the next char's `charStartTimesMs`, and the chunk's last-element values + (used to advance cumulative time) are untouched. + + Args: + alignment: Alignment dict from the API. + keys: Tuple of (chars_key, start_times_key, durations_or_end_times_key) + naming the three parallel arrays — these differ between the + WebSocket and HTTP response schemas. + """ + chars_key, starts_key, tail_key = keys + chars = alignment.get(chars_key) or [] + if chars and chars[0] == " ": + return { + chars_key: chars[1:], + starts_key: alignment.get(starts_key, [])[1:], + tail_key: alignment.get(tail_key, [])[1:], + } + return alignment + + def calculate_word_times( alignment_info: Mapping[str, Any], cumulative_time: float, @@ -790,8 +819,15 @@ class ElevenLabsTTSService(WebsocketTTSService): frame = TTSAudioRawFrame(audio, self.sample_rate, 1, context_id=received_ctx_id) await self.append_to_audio_context(received_ctx_id, frame) - if msg.get("alignment"): - alignment = msg["alignment"] + if msg.get("normalizedAlignment"): + # Use normalizedAlignment (what was actually spoken) rather than + # alignment (the input text), so word timestamps stay accurate + # when a pronunciation dictionary or text normalization rewrites + # the input. + alignment = _strip_leading_space( + msg["normalizedAlignment"], + ("chars", "charStartTimesMs", "charDurationsMs"), + ) word_times, self._partial_word, self._partial_word_start_time = ( calculate_word_times( alignment, @@ -1296,21 +1332,30 @@ class ElevenLabsHttpTTSService(TTSService): audio, self.sample_rate, 1, context_id=context_id ) - # Process alignment if present - if data and "alignment" in data: - alignment = data["alignment"] - if alignment: # Ensure alignment is not None - # Get end time of the last character in this chunk - char_end_times = alignment.get("character_end_times_seconds", []) - if char_end_times: - chunk_end_time = char_end_times[-1] - # Update to the longest end time seen so far - utterance_duration = max(utterance_duration, chunk_end_time) + # Process alignment if present. Use normalized_alignment + # (what was actually spoken) so word timestamps stay + # accurate when a pronunciation dictionary or text + # normalization rewrites the input. + if data and data.get("normalized_alignment"): + alignment = _strip_leading_space( + data["normalized_alignment"], + ( + "characters", + "character_start_times_seconds", + "character_end_times_seconds", + ), + ) + # Get end time of the last character in this chunk + char_end_times = alignment.get("character_end_times_seconds", []) + if char_end_times: + chunk_end_time = char_end_times[-1] + # Update to the longest end time seen so far + utterance_duration = max(utterance_duration, chunk_end_time) - # Calculate word timestamps - word_times = self.calculate_word_times(alignment) - if word_times: - await self.add_word_timestamps(word_times, context_id) + # Calculate word timestamps + word_times = self.calculate_word_times(alignment) + if word_times: + await self.add_word_timestamps(word_times, context_id) except json.JSONDecodeError as e: logger.warning(f"Failed to parse JSON from stream: {e}") continue diff --git a/src/pipecat/services/nvidia/llm.py b/src/pipecat/services/nvidia/llm.py index 28b635a62..7ddbe42e4 100644 --- a/src/pipecat/services/nvidia/llm.py +++ b/src/pipecat/services/nvidia/llm.py @@ -2,21 +2,44 @@ # Copyright (c) 2024-2026, Daily # # SPDX-License-Identifier: BSD 2-Clause License +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. # """NVIDIA NIM API service implementation. This module provides a service for interacting with NVIDIA's NIM (NVIDIA Inference Microservice) API while maintaining compatibility with the OpenAI-style interface. + +Refer to the NVIDIA NIM LLM API documentation for available models and usage: +https://docs.api.nvidia.com/nim/reference/llm-apis """ +from collections.abc import AsyncIterator from dataclasses import dataclass +from enum import StrEnum +from loguru import logger +from openai.types.chat import ChatCompletionChunk + +from pipecat.frames.frames import ( + LLMThoughtEndFrame, + LLMThoughtStartFrame, + LLMThoughtTextFrame, +) from pipecat.metrics.metrics import LLMTokenUsage from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.services.openai.base_llm import BaseOpenAILLMService from pipecat.services.openai.llm import OpenAILLMService +_THINK_OPEN = "" +_THINK_CLOSE = "" + + +class _ThinkTagState(StrEnum): + DETECTING = "detecting" + IN_THOUGHT = "in_thought" + CONTENT = "content" + @dataclass class NvidiaLLMSettings(BaseOpenAILLMService.Settings): @@ -28,9 +51,19 @@ class NvidiaLLMSettings(BaseOpenAILLMService.Settings): class NvidiaLLMService(OpenAILLMService): """A service for interacting with NVIDIA's NIM (NVIDIA Inference Microservice) API. - This service extends OpenAILLMService to work with NVIDIA's NIM API while maintaining - compatibility with the OpenAI-style interface. It specifically handles the difference - in token usage reporting between NIM (incremental) and OpenAI (final summary). + This service extends OpenAILLMService to work with NVIDIA's NIM API while + maintaining compatibility with the OpenAI-style interface. It handles: + + - Incremental token usage reporting (NIM sends per-chunk counts instead + of a final summary) + - Detection and filtering of leading ````/```` content for + models that emit reasoning inline before visible output (e.g. + DeepSeek-R1, some nemotron models) + - Extraction of ``reasoning_content`` from the streaming delta for models + with API-level reasoning separation (e.g. Nemotron Nano models) + + Reasoning content is emitted as ``LLMThought*Frame`` objects, keeping it + accessible to observers and logging without sending it to TTS. """ Settings = NvidiaLLMSettings @@ -39,7 +72,7 @@ class NvidiaLLMService(OpenAILLMService): def __init__( self, *, - api_key: str, + api_key: str | None = None, base_url: str = "https://integrate.api.nvidia.com/v1", model: str | None = None, settings: Settings | None = None, @@ -48,10 +81,12 @@ class NvidiaLLMService(OpenAILLMService): """Initialize the NvidiaLLMService. Args: - api_key: The API key for accessing NVIDIA's NIM API. - base_url: The base URL for NIM API. Defaults to "https://integrate.api.nvidia.com/v1". + api_key: NVIDIA API key for authentication. Required when using the + cloud endpoint. Not needed for local NIM deployments. + base_url: The base URL for NIM API. Defaults to NVIDIA's cloud endpoint. + For local deployments, pass the local address (e.g. ``http://localhost:8000/v1``). model: The model identifier to use. Defaults to - "nvidia/llama-3.1-nemotron-70b-instruct". + "nvidia/nemotron-3-nano-30b-a3b". .. deprecated:: 0.0.105 Use ``settings=NvidiaLLMService.Settings(model=...)`` instead. @@ -61,7 +96,7 @@ class NvidiaLLMService(OpenAILLMService): **kwargs: Additional keyword arguments passed to OpenAILLMService. """ # 1. Initialize default_settings with hardcoded defaults - default_settings = self.Settings(model="nvidia/llama-3.1-nemotron-70b-instruct") + default_settings = self.Settings(model="nvidia/nemotron-3-nano-30b-a3b") # 2. Apply direct init arg overrides (deprecated) if model is not None: @@ -75,6 +110,14 @@ class NvidiaLLMService(OpenAILLMService): default_settings.apply_update(settings) super().__init__(api_key=api_key, base_url=base_url, settings=default_settings, **kwargs) + + if "api.nvidia.com" in base_url and not api_key: + logger.warning( + "NvidiaLLMService: Using the cloud endpoint but no API key was provided. " + "An API key is required for the cloud endpoint. " + "Set base_url to your local NIM endpoint for local deployments." + ) + # Counters for accumulating token usage metrics self._prompt_tokens = 0 self._completion_tokens = 0 @@ -82,24 +125,202 @@ class NvidiaLLMService(OpenAILLMService): self._has_reported_prompt_tokens = False self._is_processing = False - async def _process_context(self, context: LLMContext): - """Process a context through the LLM and accumulate token usage metrics. + def _reset_response_state(self): + """Reset per-response state at the start of each LLM call. - This method overrides the parent class implementation to handle NVIDIA's - incremental token reporting style, accumulating the counts and reporting - them once at the end of processing. - - Args: - context: The context to process, containing messages and other information - needed for the LLM interaction. + Resets token accumulation counters, leading-think-tag detection state, + and reasoning-content field tracking. """ - # Reset all counters and flags at the start of processing self._prompt_tokens = 0 self._completion_tokens = 0 self._total_tokens = 0 self._has_reported_prompt_tokens = False self._is_processing = True + self._think_tag_state = _ThinkTagState.DETECTING + self._think_tag_buffer = "" + + # reasoning_content field tracking + self._has_reasoning_field = False + + async def _filter_thinking_content(self, text: str) -> str | None: + """Filter leading ```` tags from content and emit thought frames. + + Uses a three-state machine optimized for the common provider pattern + where a response either begins with a ```` block or contains no + think tags at all. It returns only visible content to the base OpenAI + processing loop while emitting hidden reasoning as ``LLMThought*Frame`` + side effects. + + - ``detecting``: Buffers the start of the stream to check for + ````. + - ``in_thought``: Inside a leading think block; emits + ``LLMThoughtTextFrame`` until ```` is found. + - ``content``: Normal content; passthrough. + + Non-reasoning models transition from ``detecting`` to ``content`` + on the first chunk with zero buffering overhead after that. + + Args: + text: The text content from the LLM to filter. + + Returns: + The non-reasoning content that should continue through the base + OpenAI content path, or ``None`` if this chunk should not emit + normal content. + + """ + if self._think_tag_state == _ThinkTagState.CONTENT: + return text + + self._think_tag_buffer += text + + if self._think_tag_state == _ThinkTagState.DETECTING: + if len(self._think_tag_buffer) < len(_THINK_OPEN): + if _THINK_OPEN.startswith(self._think_tag_buffer): + return None + self._think_tag_state = _ThinkTagState.CONTENT + passthrough = self._think_tag_buffer + self._think_tag_buffer = "" + return passthrough + + if self._think_tag_buffer.startswith(_THINK_OPEN): + self._think_tag_state = _ThinkTagState.IN_THOUGHT + await self.push_frame(LLMThoughtStartFrame()) + self._think_tag_buffer = self._think_tag_buffer[len(_THINK_OPEN) :] + else: + self._think_tag_state = _ThinkTagState.CONTENT + passthrough = self._think_tag_buffer + self._think_tag_buffer = "" + return passthrough + + if self._think_tag_state == _ThinkTagState.IN_THOUGHT: + idx = self._think_tag_buffer.find(_THINK_CLOSE) + if idx != -1: + thought = self._think_tag_buffer[:idx] + if thought: + await self.push_frame(LLMThoughtTextFrame(text=thought)) + await self.push_frame(LLMThoughtEndFrame()) + remainder = self._think_tag_buffer[idx + len(_THINK_CLOSE) :] + self._think_tag_buffer = "" + self._think_tag_state = _ThinkTagState.CONTENT + return remainder or None + else: + safe_end = len(self._think_tag_buffer) - len(_THINK_CLOSE) + 1 + if safe_end > 0: + await self.push_frame( + LLMThoughtTextFrame(text=self._think_tag_buffer[:safe_end]) + ) + self._think_tag_buffer = self._think_tag_buffer[safe_end:] + return None + + async def _flush_reasoning_state(self): + """Flush buffered reasoning state at normal stream completion. + + Emits any buffered trailing thought text, closes open thought frames, + and forwards any buffered pre-content text that was held while deciding + whether the stream began with ````. + """ + if self._think_tag_state == _ThinkTagState.IN_THOUGHT: + if self._think_tag_buffer: + await self.push_frame(LLMThoughtTextFrame(text=self._think_tag_buffer)) + await self.push_frame(LLMThoughtEndFrame()) + elif self._think_tag_state == _ThinkTagState.DETECTING and self._think_tag_buffer: + await super()._push_llm_text(self._think_tag_buffer) + + self._think_tag_buffer = "" + self._think_tag_state = _ThinkTagState.CONTENT + + if self._has_reasoning_field: + await self.push_frame(LLMThoughtEndFrame()) + self._has_reasoning_field = False + + async def get_chat_completions(self, context: LLMContext) -> AsyncIterator[ChatCompletionChunk]: + """Wrap the chat completion stream to handle ``reasoning_content``. + + Models with API-level reasoning separation (e.g. Nemotron Nano) + include a ``reasoning_content`` field on the streaming delta. This + wrapper extracts those chunks and emits them as ``LLMThought*Frame`` + objects. It also rewrites streamed ``delta.content`` so leading + ```` sections are removed before the base OpenAI loop processes + visible content. + + Args: + context: The LLM context for the completion request. + + Returns: + An async iterator of chat completion chunks where + ``reasoning_content`` has been emitted as ``LLMThought*Frame`` + side effects. + """ + stream = await super().get_chat_completions(context) + return self._handle_reasoning_content(stream) + + async def _handle_reasoning_content( + self, stream: AsyncIterator[ChatCompletionChunk] + ) -> AsyncIterator[ChatCompletionChunk]: + """Handle ``reasoning_content`` and leading ```` tags in a chunk stream. + + Inspects each chunk for a ``reasoning_content`` field on the delta and + emits ``LLMThoughtStartFrame`` / ``LLMThoughtTextFrame`` / + ``LLMThoughtEndFrame`` as side effects. It also strips ```` + blocks from ``delta.content`` before yielding the chunk so the base + OpenAI loop only sees user-facing content. Every chunk is still yielded + so the base streaming loop can process metadata such as token usage, + model name, tool calls, and audio transcripts. + + Notes: + Stream cleanup is owned by the base OpenAI processing loop + (``BaseOpenAILLMService._process_context``), which wraps the stream + in its own closing context manager. + + Args: + stream: The original chat completion stream. + + Yields: + Chat completion chunks with any leading ```` content removed + from ``delta.content`` before they reach the base OpenAI loop. + """ + async for chunk in stream: + if chunk.choices and len(chunk.choices) > 0 and chunk.choices[0].delta: + delta = chunk.choices[0].delta + rc = getattr(delta, "reasoning_content", None) + if rc: + if not self._has_reasoning_field: + self._has_reasoning_field = True + await self.push_frame(LLMThoughtStartFrame()) + await self.push_frame(LLMThoughtTextFrame(text=rc)) + elif self._has_reasoning_field and delta.content: + await self.push_frame(LLMThoughtEndFrame()) + self._has_reasoning_field = False + + if delta.content: + delta.content = await self._filter_thinking_content(delta.content) + yield chunk + + await self._flush_reasoning_state() + + async def _process_context(self, context: LLMContext): + """Process a context through the LLM and accumulate token usage metrics. + + Delegates to the base OpenAI streaming loop while adding + NVIDIA-specific behavior: + + - ``reasoning_content`` and leading ```` content are + intercepted via the ``get_chat_completions`` stream wrapper and + emitted as + ``LLMThought*Frame`` objects. + - Incremental token counts are accumulated and reported as final + totals. + + Args: + context: The context to process, containing messages and other + information needed for the LLM interaction. + """ + self._reset_response_state() + + # Wrap in try/finally to guarantee accumulated token metrics are + # reported and _is_processing is cleared even on cancellation. try: await super()._process_context(context) finally: diff --git a/src/pipecat/services/sarvam/stt.py b/src/pipecat/services/sarvam/stt.py index faadd914b..7c949ac59 100644 --- a/src/pipecat/services/sarvam/stt.py +++ b/src/pipecat/services/sarvam/stt.py @@ -91,6 +91,7 @@ class ModelConfig: supports_prompt: Whether the model accepts prompt parameter. supports_mode: Whether the model accepts mode parameter. supports_language: Whether the model accepts language parameter. + supports_vad_params: Whether the model accepts fine-grained VAD parameters. default_language: Default language code (None = auto-detect). default_mode: Default mode (None = not applicable). use_translate_endpoint: Whether to use speech_to_text_translate_streaming endpoint. @@ -100,6 +101,7 @@ class ModelConfig: supports_prompt: bool supports_mode: bool supports_language: bool + supports_vad_params: bool default_language: str | None default_mode: str | None use_translate_endpoint: bool @@ -111,6 +113,7 @@ MODEL_CONFIGS: dict[str, ModelConfig] = { supports_prompt=False, supports_mode=False, supports_language=True, + supports_vad_params=False, default_language="unknown", default_mode=None, use_translate_endpoint=False, @@ -120,6 +123,7 @@ MODEL_CONFIGS: dict[str, ModelConfig] = { supports_prompt=True, supports_mode=False, supports_language=False, + supports_vad_params=False, default_language=None, # Auto-detects language default_mode=None, use_translate_endpoint=True, @@ -129,6 +133,7 @@ MODEL_CONFIGS: dict[str, ModelConfig] = { supports_prompt=False, supports_mode=True, supports_language=True, + supports_vad_params=True, default_language="unknown", default_mode="transcribe", use_translate_endpoint=False, @@ -146,11 +151,43 @@ class SarvamSTTSettings(STTSettings): Only applicable to models that support prompts (e.g., saaras:v2.5). vad_signals: Enable VAD signals in response. high_vad_sensitivity: Enable high VAD sensitivity. + positive_speech_threshold: VAD probability threshold (0.0-1.0) above which + a frame is considered speech. Only for saaras:v3. + negative_speech_threshold: VAD probability threshold (0.0-1.0) below which + a frame is considered silence. Only for saaras:v3. + min_speech_frames: Minimum consecutive speech frames to start a speech + segment. Only for saaras:v3. + first_turn_min_speech_frames: Minimum speech frames for the first user + turn. Only for saaras:v3. + negative_frames_count: Number of silence frames within the window to end + a speech segment. Only for saaras:v3. + negative_frames_window: Sliding window size (in frames) for counting + negative frames. Only for saaras:v3. + start_speech_volume_threshold: Volume level (dB) below which audio is + too quiet to be speech. Only for saaras:v3. + interrupt_min_speech_frames: Minimum speech frames to register a + barge-in/interruption. Only for saaras:v3. + pre_speech_pad_frames: Number of audio frames to prepend before detected + speech onset. Only for saaras:v3. + num_initial_ignored_frames: Number of leading audio frames to skip at + connection start. Only for saaras:v3. """ prompt: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) vad_signals: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) high_vad_sensitivity: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + positive_speech_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + negative_speech_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + min_speech_frames: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + first_turn_min_speech_frames: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + negative_frames_count: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + negative_frames_window: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + start_speech_volume_threshold: float | None | _NotGiven = field( + default_factory=lambda: NOT_GIVEN + ) + interrupt_min_speech_frames: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + pre_speech_pad_frames: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + num_initial_ignored_frames: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) class SarvamSTTService(STTService): @@ -190,7 +227,7 @@ class SarvamSTTService(STTService): mode: Mode of operation for saaras:v3 models only. Options: transcribe, translate, verbatim, translit, codemix. Defaults to "transcribe" for saaras:v3. vad_signals: Enable VAD signals in response. Defaults to None. - high_vad_sensitivity: Enable high VAD (Voice Activity Detection) sensitivity. Defaults to None. + high_vad_sensitivity: Enable high VAD sensitivity. Defaults to None. """ language: Language | None = None @@ -244,11 +281,21 @@ class SarvamSTTService(STTService): """ # --- 1. Hardcoded defaults --- default_settings = self.Settings( - model="saarika:v2.5", + model="saaras:v3", language=None, prompt=None, vad_signals=None, high_vad_sensitivity=None, + positive_speech_threshold=None, + negative_speech_threshold=None, + min_speech_frames=None, + first_turn_min_speech_frames=None, + negative_frames_count=None, + negative_frames_window=None, + start_speech_volume_threshold=None, + interrupt_min_speech_frames=None, + pre_speech_pad_frames=None, + num_initial_ignored_frames=None, ) # --- 2. Deprecated direct-arg overrides --- @@ -289,6 +336,26 @@ class SarvamSTTService(STTService): f"Model '{resolved_model}' does not support language parameter (auto-detects language)." ) + if not self._config.supports_vad_params: + vad_param_names = [ + "positive_speech_threshold", + "negative_speech_threshold", + "min_speech_frames", + "first_turn_min_speech_frames", + "negative_frames_count", + "negative_frames_window", + "start_speech_volume_threshold", + "interrupt_min_speech_frames", + "pre_speech_pad_frames", + "num_initial_ignored_frames", + ] + for param_name in vad_param_names: + if getattr(default_settings, param_name) is not None: + raise ValueError( + f"Model '{resolved_model}' does not support {param_name} parameter. " + f"Fine-grained VAD parameters are only supported by saaras:v3." + ) + # Resolve mode default from model config if mode is None: mode = self._config.default_mode @@ -393,10 +460,44 @@ class SarvamSTTService(STTService): f"Model '{self._settings.model}' does not support prompt parameter." ) + if isinstance(delta, self.Settings) and not self._config.supports_vad_params: + vad_param_names = [ + "positive_speech_threshold", + "negative_speech_threshold", + "min_speech_frames", + "first_turn_min_speech_frames", + "negative_frames_count", + "negative_frames_window", + "start_speech_volume_threshold", + "interrupt_min_speech_frames", + "pre_speech_pad_frames", + "num_initial_ignored_frames", + ] + for param_name in vad_param_names: + val = getattr(delta, param_name, NOT_GIVEN) + if is_given(val) and val is not None: + raise ValueError( + f"Model '{self._settings.model}' does not support {param_name} " + f"parameter. Fine-grained VAD parameters are only supported by saaras:v3." + ) + changed = await super()._update_settings(delta) - # Language and prompt are WebSocket connect-time parameters; reconnect to apply. - reconnect_fields = {"language", "prompt"} + # These are all WebSocket connect-time parameters; reconnect to apply. + reconnect_fields = { + "language", + "prompt", + "positive_speech_threshold", + "negative_speech_threshold", + "min_speech_frames", + "first_turn_min_speech_frames", + "negative_frames_count", + "negative_frames_window", + "start_speech_volume_threshold", + "interrupt_min_speech_frames", + "pre_speech_pad_frames", + "num_initial_ignored_frames", + } if changed.keys() & reconnect_fields: await self._disconnect() await self._connect() @@ -535,6 +636,24 @@ class SarvamSTTService(STTService): "true" if self._settings.high_vad_sensitivity else "false" ) + # Fine-grained VAD parameters (saaras:v3 only, sent as strings per SDK spec) + if self._config.supports_vad_params: + _vad_params = { + "positive_speech_threshold": self._settings.positive_speech_threshold, + "negative_speech_threshold": self._settings.negative_speech_threshold, + "min_speech_frames": self._settings.min_speech_frames, + "first_turn_min_speech_frames": self._settings.first_turn_min_speech_frames, + "negative_frames_count": self._settings.negative_frames_count, + "negative_frames_window": self._settings.negative_frames_window, + "start_speech_volume_threshold": self._settings.start_speech_volume_threshold, + "interrupt_min_speech_frames": self._settings.interrupt_min_speech_frames, + "pre_speech_pad_frames": self._settings.pre_speech_pad_frames, + "num_initial_ignored_frames": self._settings.num_initial_ignored_frames, + } + for k, v in _vad_params.items(): + if v is not None: + connect_kwargs[k] = str(v) + # Add language_code for models that support it language_string = self._get_language_string() if language_string is not None: diff --git a/src/pipecat/services/smallest/tts.py b/src/pipecat/services/smallest/tts.py index 33c492c01..8bc22035c 100644 --- a/src/pipecat/services/smallest/tts.py +++ b/src/pipecat/services/smallest/tts.py @@ -125,7 +125,7 @@ class SmallestTTSService(InterruptibleTTSService): self, *, api_key: str, - base_url: str = "wss://waves-api.smallest.ai", + base_url: str = "wss://api.smallest.ai", sample_rate: int | None = None, settings: Settings | None = None, **kwargs, @@ -174,6 +174,10 @@ class SmallestTTSService(InterruptibleTTSService): """ return True + async def flush_audio(self, context_id: str | None = None): + """Flush any pending audio data.""" + logger.trace(f"{self}: flushing audio") + def language_to_service_language(self, language: Language) -> str | None: """Convert a Language enum to Smallest service language format. @@ -217,7 +221,7 @@ class SmallestTTSService(InterruptibleTTSService): def _build_websocket_url(self) -> str: """Build the WebSocket URL from base URL and model.""" - return f"{self._base_url}/api/v1/{self._settings.model}/get_speech/stream" + return f"{self._base_url}/waves/v1/{self._settings.model}/get_speech/stream" async def start(self, frame: StartFrame): """Start the Smallest TTS service. @@ -350,17 +354,15 @@ class SmallestTTSService(InterruptibleTTSService): await self._send_keepalive() async def _send_keepalive(self): - """Send a flush message to keep the connection alive.""" + """Send a silent message to keep the WebSocket connection alive.""" if self._websocket and self._websocket.state is State.OPEN: - msg = {"flush": True} + msg = { + "text": " ", + "voice_id": self._settings.voice, + "language": self._settings.language, + } await self._websocket.send(json.dumps(msg)) - async def flush_audio(self, context_id: str | None = None): - """Flush any pending audio synthesis.""" - if not self._websocket or self._websocket.state is State.CLOSED: - return - await self._get_websocket().send(json.dumps({"flush": True})) - async def _receive_messages(self): """Receive and process messages from the Smallest WebSocket API.""" async for message in self._get_websocket(): diff --git a/src/pipecat/services/stt_latency.py b/src/pipecat/services/stt_latency.py index 5ffd798de..7d285e353 100644 --- a/src/pipecat/services/stt_latency.py +++ b/src/pipecat/services/stt_latency.py @@ -57,3 +57,4 @@ WHISPER_TTFS_P99: float = DEFAULT_TTFS_P99 # No benchmark available yet; using conservative default MISTRAL_TTFS_P99: float = DEFAULT_TTFS_P99 SMALLEST_TTFS_P99: float = DEFAULT_TTFS_P99 +XAI_TTFS_P99: float = DEFAULT_TTFS_P99 diff --git a/src/pipecat/services/tts_service.py b/src/pipecat/services/tts_service.py index fa5981802..14e478c5e 100644 --- a/src/pipecat/services/tts_service.py +++ b/src/pipecat/services/tts_service.py @@ -293,6 +293,7 @@ class TTSService(AIService): self._processing_text: bool = False self._tts_contexts: dict[str, TTSContext] = {} self._streamed_text: str = "" + self._sent_non_whitespace_in_context: bool = False self._text_aggregation_metrics_started: bool = False # Word timestamp state @@ -684,6 +685,7 @@ class TTSService(AIService): # Reset aggregator state self._processing_text = False + self._sent_non_whitespace_in_context = False if isinstance(frame, LLMFullResponseEndFrame): if self._push_text_frames: # Route through the serialization queue so the frame is @@ -698,6 +700,8 @@ class TTSService(AIService): elif isinstance(frame, TTSSpeakFrame): # Store if we were processing text or not so we can set it back. processing_text = self._processing_text + saved_sent_non_whitespace = self._sent_non_whitespace_in_context + self._sent_non_whitespace_in_context = False # TTSSpeakFrame is independent — temporarily clear the turn context # so create_context_id() generates a fresh UUID for this utterance. saved_turn_context_id = self._turn_context_id @@ -718,6 +722,7 @@ class TTSService(AIService): # the TTS. We pause to avoid audio overlapping. await self._maybe_pause_frame_processing() self._turn_context_id = saved_turn_context_id + self._sent_non_whitespace_in_context = saved_sent_non_whitespace self._processing_text = processing_text elif isinstance(frame, TTSUpdateSettingsFrame): if frame.service is not None and frame.service is not self: @@ -844,6 +849,7 @@ class TTSService(AIService): async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection): self._processing_text = False + self._sent_non_whitespace_in_context = False await self._text_aggregator.handle_interruption() for filter in self._text_filters: await filter.handle_interruption() @@ -901,13 +907,22 @@ class TTSService(AIService): await self.push_frame(src_frame) return - # Remove leading newlines only - text = text.lstrip("\n") - - # Don't send only whitespace. This causes problems for some TTS models. But also don't - # strip all whitespace, as whitespace can influence prosody. - if not text.strip(): - return + # Whitespace gating depends on aggregation mode: + # - Token streaming: drop all leading whitespace at the start of a context, as + # nothing substantive has been sent yet for it to attach to. Once a non-whitespace + # token has been sent, send whitespace as-is since it can influence prosody between + # non-whitespace tokens. + # + # - Sentence aggregation: strip leading newlines only and drop pure-whitespace frames. + if self._is_streaming_tokens: + if not self._sent_non_whitespace_in_context: + text = text.lstrip() + if not text: + return + else: + text = text.lstrip("\n") + if not text.strip(): + return # This is just a flag that indicates if we sent something to the TTS # service. It will be cleared if we sent text because of a TTSSpeakFrame @@ -929,9 +944,15 @@ class TTSService(AIService): await filter.reset_interruption() text = await filter.filter(text) - if not text.strip(): - if not self._is_streaming_tokens: - await self.stop_processing_metrics() + # Post-filter whitespace gate. Mirrors the pre-filter logic so filter + # output that collapses to whitespace-only is handled consistently. + if self._is_streaming_tokens: + # If empty, or only-whitespace and we haven't sent any non-whitespace, skip. + if not text or (not text.strip() and not self._sent_non_whitespace_in_context): + return + self._sent_non_whitespace_in_context = True + elif not text.strip(): + await self.stop_processing_metrics() return # Create context ID and store metadata diff --git a/src/pipecat/services/xai/stt.py b/src/pipecat/services/xai/stt.py new file mode 100644 index 000000000..df8406fc2 --- /dev/null +++ b/src/pipecat/services/xai/stt.py @@ -0,0 +1,411 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""xAI speech-to-text service implementation. + +This module provides integration with xAI's real-time speech-to-text WebSocket +API documented at https://docs.x.ai/developers/rest-api-reference/inference/voice. +""" + +import asyncio +import json +from collections.abc import AsyncGenerator +from dataclasses import dataclass, field +from typing import Any +from urllib.parse import urlencode + +from loguru import logger + +from pipecat import version as pipecat_version +from pipecat.frames.frames import ( + CancelFrame, + EndFrame, + Frame, + InterimTranscriptionFrame, + StartFrame, + TranscriptionFrame, +) +from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven +from pipecat.services.stt_latency import XAI_TTFS_P99 +from pipecat.services.stt_service import WebsocketSTTService +from pipecat.transcriptions.language import Language, resolve_language +from pipecat.utils.time import time_now_iso8601 +from pipecat.utils.tracing.service_decorators import traced_stt + +try: + from websockets.asyncio.client import connect as websocket_connect + from websockets.protocol import State +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error('In order to use xAI STT, you need to `pip install "pipecat-ai[xai]"`.') + raise Exception(f"Missing module: {e}") + + +def language_to_xai_stt_language(language: Language) -> str | None: + """Convert a Language enum to the xAI STT language code. + + xAI STT accepts two-letter language codes (e.g. ``en``, ``fr``, ``de``, + ``ja``). When set, the server applies Inverse Text Normalization. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding xAI STT language code, or None if not supported. + """ + LANGUAGE_MAP = { + Language.AR: "ar", + Language.BN: "bn", + Language.DE: "de", + Language.EN: "en", + Language.ES: "es", + Language.FR: "fr", + Language.HI: "hi", + Language.ID: "id", + Language.IT: "it", + Language.JA: "ja", + Language.KO: "ko", + Language.PT: "pt", + Language.RU: "ru", + Language.TR: "tr", + Language.VI: "vi", + Language.ZH: "zh", + } + + return resolve_language(language, LANGUAGE_MAP, use_base_code=True) + + +@dataclass +class XAISTTSettings(STTSettings): + """Settings for XAISTTService. + + Parameters: + interim_results: When True, partial transcripts are emitted + approximately every 500ms. + endpointing: Silence duration in milliseconds that triggers a + speech-final event. Range 0-5000. Server default is 10ms. + multichannel: When True, transcribes each interleaved channel + independently. Requires ``channels`` >= 2. + channels: Number of interleaved channels (2-8). Required when + ``multichannel`` is True. + diarize: When True, the server attaches a ``speaker`` field to each + word identifying the detected speaker. + """ + + interim_results: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + endpointing: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + multichannel: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + channels: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + diarize: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + +class XAISTTService(WebsocketSTTService): + """xAI real-time speech-to-text service. + + Streams audio to xAI's WebSocket STT endpoint and emits interim and final + transcription frames. The ``XAI_API_KEY`` is passed directly as a Bearer + token on the WebSocket handshake. + + The connection is persistent: audio is streamed continuously and the + server emits ``transcript.partial`` events with ``is_final`` and + ``speech_final`` flags to mark utterance boundaries. If the connection + drops mid-session, the base class reconnects automatically. + """ + + Settings = XAISTTSettings + _settings: Settings + + def __init__( + self, + *, + api_key: str, + ws_url: str = "wss://api.x.ai/v1/stt", + sample_rate: int = 16000, + encoding: str = "pcm", + settings: Settings | None = None, + ttfs_p99_latency: float | None = XAI_TTFS_P99, + **kwargs, + ): + """Initialize the xAI STT service. + + Args: + api_key: xAI API key (used as Bearer for the WebSocket handshake). + ws_url: WebSocket endpoint URL. Defaults to ``wss://api.x.ai/v1/stt``. + sample_rate: Audio sample rate in Hz. Supported values: 8000, + 16000, 22050, 24000, 44100, 48000. Defaults to 16000. + encoding: Audio encoding. One of ``"pcm"`` (signed 16-bit LE), + ``"mulaw"``, or ``"alaw"``. Defaults to ``"pcm"``. + settings: Runtime-updatable settings overriding defaults. + ttfs_p99_latency: P99 latency from speech end to final transcript + in seconds. See https://github.com/pipecat-ai/stt-benchmark. + **kwargs: Additional arguments passed to WebsocketSTTService. + """ + default_settings = self.Settings( + model=None, + language=Language.EN, + interim_results=True, + endpointing=None, + multichannel=None, + channels=None, + diarize=None, + ) + if settings is not None: + default_settings.apply_update(settings) + + super().__init__( + sample_rate=sample_rate, + settings=default_settings, + ttfs_p99_latency=ttfs_p99_latency, + **kwargs, + ) + + self._api_key = api_key + self._ws_url = ws_url + self._encoding = encoding + + self._receive_task: asyncio.Task | None = None + self._session_ready = asyncio.Event() + + def can_generate_metrics(self) -> bool: + """Check if the service can generate metrics. + + Returns: + True if metrics generation is supported. + """ + return True + + def language_to_service_language(self, language: Language) -> str | None: + """Convert a Language enum to the xAI STT language code.""" + return language_to_xai_stt_language(language) + + async def _update_settings(self, delta: Settings) -> dict[str, Any]: + """Apply a settings delta and reconnect to apply changes. + + xAI STT configures the session via WebSocket query parameters, so any + change requires a fresh connection. + """ + changed = await super()._update_settings(delta) + if not changed: + return changed + await self._disconnect() + await self._connect() + return changed + + async def start(self, frame: StartFrame): + """Start the speech-to-text service.""" + await super().start(frame) + await self._connect() + + async def stop(self, frame: EndFrame): + """Stop the speech-to-text service.""" + await super().stop(frame) + await self._disconnect() + + async def cancel(self, frame: CancelFrame): + """Cancel the speech-to-text service.""" + await super().cancel(frame) + await self._disconnect() + + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Forward raw audio bytes to the xAI STT WebSocket. + + Transcription frames are pushed from the receive task, not yielded + from this coroutine. + """ + if self._websocket and self._websocket.state is State.OPEN and self._session_ready.is_set(): + try: + await self._websocket.send(audio) + except Exception as e: + await self.push_error(error_msg=f"xAI STT send failed: {e}", exception=e) + yield None + + def _build_ws_url(self) -> str: + """Build the WebSocket URL with session query parameters.""" + s = self._settings + params: dict[str, Any] = { + "sample_rate": self.sample_rate, + "encoding": self._encoding, + } + + if s.language is not None: + params["language"] = s.language + + optional_fields = { + "interim_results": s.interim_results, + "endpointing": s.endpointing, + "multichannel": s.multichannel, + "channels": s.channels, + "diarize": s.diarize, + } + for key, val in optional_fields.items(): + if val is None: + continue + if isinstance(val, bool): + params[key] = str(val).lower() + else: + params[key] = val + + return f"{self._ws_url}?{urlencode(params)}" + + async def _connect(self): + """Establish the WebSocket connection and start the receive task.""" + await super()._connect() + await self._connect_websocket() + if self._websocket and not self._receive_task: + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) + + async def _disconnect(self): + """Tear down the WebSocket connection and cancel the receive task.""" + await super()._disconnect() + try: + if self._websocket and self._websocket.state is State.OPEN: + await self._websocket.send(json.dumps({"type": "audio.done"})) + except Exception as e: + logger.debug(f"{self} error sending audio.done during disconnect: {e}") + + if self._receive_task: + await self.cancel_task(self._receive_task) + self._receive_task = None + + await self._disconnect_websocket() + + async def _connect_websocket(self): + """Open a WebSocket connection to the xAI STT endpoint.""" + try: + if self._websocket and self._websocket.state is State.OPEN: + return + + logger.debug("Connecting to xAI STT WebSocket") + self._session_ready.clear() + + ws_url = self._build_ws_url() + headers = { + "Authorization": f"Bearer {self._api_key}", + "User-Agent": f"xAI/1.0 (integration=Pipecat/{pipecat_version()})", + } + self._websocket = await websocket_connect(ws_url, additional_headers=headers) + await self._call_event_handler("on_connected") + logger.debug(f"{self} connected to xAI STT WebSocket") + except Exception as e: + await self.push_error(error_msg=f"Unable to connect to xAI STT: {e}", exception=e) + raise + + async def _disconnect_websocket(self): + """Close the WebSocket connection.""" + try: + if self._websocket: + logger.debug("Disconnecting from xAI STT WebSocket") + await self._websocket.close() + except Exception as e: + await self.push_error(error_msg=f"Error closing xAI STT websocket: {e}", exception=e) + finally: + self._websocket = None + self._session_ready.clear() + await self._call_event_handler("on_disconnected") + + async def _receive_messages(self): + """Receive and dispatch xAI STT WebSocket messages.""" + if not self._websocket: + raise Exception("Websocket not connected") + async for message in self._websocket: + try: + data = json.loads(message) + except json.JSONDecodeError: + logger.warning(f"{self} received non-JSON message: {message}") + continue + await self._handle_message(data) + + async def _handle_message(self, message: dict[str, Any]): + """Branch on xAI STT event type.""" + msg_type = message.get("type") + + if msg_type == "transcript.created": + self._session_ready.set() + logger.debug(f"{self} xAI STT session ready") + elif msg_type == "transcript.partial": + await self._handle_transcript(message) + elif msg_type == "transcript.done": + if message.get("text"): + await self._push_final_transcript(message, speech_final=True) + elif msg_type == "error": + await self.push_error( + error_msg=f"xAI STT error: {message.get('message', message)}", + exception=Exception(message), + ) + else: + logger.debug(f"{self} unhandled xAI STT message: {message}") + + async def _handle_transcript(self, message: dict[str, Any]): + text = message.get("text", "") + if not text: + return + + is_final = bool(message.get("is_final")) + speech_final = bool(message.get("speech_final")) + language = self._language_for_frame() + + if is_final: + await self._push_final_transcript( + message, speech_final=speech_final, language=language, text=text + ) + else: + await self.push_frame( + InterimTranscriptionFrame( + text, + self._user_id, + time_now_iso8601(), + language, + result=message, + ) + ) + + async def _push_final_transcript( + self, + message: dict[str, Any], + *, + speech_final: bool, + language: Language | None = None, + text: str | None = None, + ): + text = text if text is not None else message.get("text", "") + if not text: + return + language = language if language is not None else self._language_for_frame() + + await self.push_frame( + TranscriptionFrame( + text, + self._user_id, + time_now_iso8601(), + language, + result=message, + finalized=speech_final, + ) + ) + await self._trace_transcription(text, True, language) + if speech_final: + await self.stop_processing_metrics() + + def _language_for_frame(self) -> Language: + """Return a Language enum suitable for transcription frames. + + Settings stores the service-specific string (e.g. ``"en"``); frames + carry the enum value. + """ + lang = self._settings.language + if isinstance(lang, Language): + return lang + if isinstance(lang, str): + try: + return Language(lang) + except ValueError: + return Language.EN + return Language.EN + + @traced_stt + async def _trace_transcription(self, transcript: str, is_final: bool, language: Language): + """Record transcription event for tracing.""" + pass diff --git a/src/pipecat/services/xai/tts.py b/src/pipecat/services/xai/tts.py index 17f67cf9a..f99772baa 100644 --- a/src/pipecat/services/xai/tts.py +++ b/src/pipecat/services/xai/tts.py @@ -6,22 +6,48 @@ """xAI text-to-speech service implementation. -Uses xAI's HTTP TTS endpoint documented at: -https://docs.x.ai/developers/model-capabilities/audio/text-to-speech +Provides two TTS services against xAI's voice API: + +- :class:`XAIHttpTTSService` uses the batch HTTP endpoint at + ``https://api.x.ai/v1/tts``. +- :class:`XAITTSService` uses the streaming WebSocket endpoint at + ``wss://api.x.ai/v1/tts``. + +See https://docs.x.ai/developers/rest-api-reference/inference/voice. """ +import base64 +import json from collections.abc import AsyncGenerator from dataclasses import dataclass +from typing import Any +from urllib.parse import urlencode import aiohttp from loguru import logger -from pipecat.frames.frames import ErrorFrame, Frame, TTSAudioRawFrame +from pipecat.frames.frames import ( + CancelFrame, + EndFrame, + ErrorFrame, + Frame, + StartFrame, + TTSAudioRawFrame, + TTSStoppedFrame, +) from pipecat.services.settings import TTSSettings -from pipecat.services.tts_service import TTSService +from pipecat.services.tts_service import InterruptibleTTSService, TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts +try: + from websockets.asyncio.client import connect as websocket_connect + from websockets.protocol import State +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use XAITTSService, you need to `pip install pipecat-ai[xai]`.") + raise Exception(f"Missing module: {e}") + def language_to_xai_language(language: Language) -> str | None: """Convert a Language enum to xAI language code. @@ -214,3 +240,249 @@ class XAIHttpTTSService(TTSService): ) except Exception as e: yield ErrorFrame(error=f"Unknown error occurred: {e}") + + +@dataclass +class XAIWebsocketTTSSettings(TTSSettings): + """Settings for XAITTSService (WebSocket streaming).""" + + pass + + +class XAITTSService(InterruptibleTTSService): + """xAI streaming text-to-speech service. + + Connects to xAI's WebSocket TTS endpoint and streams audio chunks back as + they are synthesized. Text can be sent incrementally via ``text.delta`` + messages and each utterance is terminated with ``text.done``. The server + responds with ``audio.delta`` chunks followed by an ``audio.done`` message. + + Audio parameters (voice, language, codec, sample rate, bit rate) are passed + as query string parameters on the WebSocket URL; changing any of them at + runtime reconnects the WebSocket. + """ + + Settings = XAIWebsocketTTSSettings + _settings: Settings + + def __init__( + self, + *, + api_key: str, + base_url: str = "wss://api.x.ai/v1/tts", + sample_rate: int | None = None, + codec: str = "pcm", + settings: Settings | None = None, + **kwargs, + ): + """Initialize the xAI WebSocket TTS service. + + Args: + api_key: xAI API key for authentication. + base_url: xAI TTS WebSocket endpoint. Defaults to + ``wss://api.x.ai/v1/tts``. + sample_rate: Output audio sample rate in Hz. If None, uses the + pipeline default. + codec: Output audio codec. One of ``pcm``, ``wav``, ``mulaw``, + ``alaw``. Defaults to ``pcm`` so emitted ``TTSAudioRawFrame`` + objects need no decoding downstream. + settings: Runtime-updatable settings. + **kwargs: Additional arguments passed to parent + ``InterruptibleTTSService``. + """ + default_settings = self.Settings( + model=None, + voice="eve", + language=Language.EN, + ) + + if settings is not None: + default_settings.apply_update(settings) + + super().__init__( + push_start_frame=True, + push_stop_frames=True, + sample_rate=sample_rate, + settings=default_settings, + **kwargs, + ) + + self._api_key = api_key + self._base_url = base_url + self._codec = codec + self._receive_task = None + + def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics.""" + return True + + def language_to_service_language(self, language: Language) -> str | None: + """Convert a Language enum to xAI language format.""" + return language_to_xai_language(language) + + async def start(self, frame: StartFrame): + """Start the xAI WebSocket TTS service.""" + await super().start(frame) + await self._connect() + + async def stop(self, frame: EndFrame): + """Stop the xAI WebSocket TTS service.""" + await super().stop(frame) + await self._disconnect() + + async def cancel(self, frame: CancelFrame): + """Cancel the xAI WebSocket TTS service.""" + await super().cancel(frame) + await self._disconnect() + + async def _connect(self): + await super()._connect() + + await self._connect_websocket() + + if self._websocket and not self._receive_task: + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) + + async def _disconnect(self): + await super()._disconnect() + + if self._receive_task: + await self.cancel_task(self._receive_task) + self._receive_task = None + + await self._disconnect_websocket() + + async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]: + """Apply a settings delta. Reconnects if any URL-baked field changes.""" + changed = await super()._update_settings(delta) + + if changed: + await self._disconnect() + await self._connect() + + return changed + + def _build_url(self) -> str: + language = self._settings.language + if isinstance(language, Language): + language_value = language_to_xai_language(language) or language.value + else: + language_value = str(language) if language is not None else "auto" + + params: dict[str, Any] = { + "voice": self._settings.voice, + "language": language_value, + "codec": self._codec, + "sample_rate": self.sample_rate, + } + return f"{self._base_url}?{urlencode(params)}" + + async def _connect_websocket(self): + try: + if self._websocket and self._websocket.state is State.OPEN: + return + + logger.debug("Connecting to xAI TTS") + + url = self._build_url() + headers = {"Authorization": f"Bearer {self._api_key}"} + self._websocket = await websocket_connect(url, additional_headers=headers) + + await self._call_event_handler("on_connected") + except Exception as e: + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) + self._websocket = None + await self._call_event_handler("on_connection_error", f"{e}") + + async def _disconnect_websocket(self): + try: + await self.stop_all_metrics() + + if self._websocket: + logger.debug("Disconnecting from xAI TTS") + await self._websocket.close() + except Exception as e: + await self.push_error(error_msg=f"Error disconnecting from xAI TTS: {e}", exception=e) + finally: + self._websocket = None + await self._call_event_handler("on_disconnected") + + def _get_websocket(self): + if self._websocket: + return self._websocket + raise Exception("Websocket not connected") + + async def flush_audio(self, context_id: str | None = None): + """Signal end-of-utterance so xAI begins synthesizing what it has buffered.""" + if not self._websocket or self._websocket.state is State.CLOSED: + return + await self._get_websocket().send(json.dumps({"type": "text.done"})) + + async def _receive_messages(self): + async for message in self._get_websocket(): + if isinstance(message, bytes): + logger.warning(f"{self}: unexpected binary frame from xAI TTS") + continue + try: + msg = json.loads(message) + except json.JSONDecodeError: + logger.error(f"{self}: invalid JSON message: {message}") + continue + + msg_type = msg.get("type") + context_id = self.get_active_audio_context_id() + + if msg_type == "audio.delta": + audio_b64 = msg.get("delta") + if not audio_b64: + continue + audio = base64.b64decode(audio_b64) + await self.stop_ttfb_metrics() + if context_id: + frame = TTSAudioRawFrame( + audio=audio, + sample_rate=self.sample_rate, + num_channels=1, + context_id=context_id, + ) + await self.append_to_audio_context(context_id, frame) + elif msg_type == "audio.done": + await self.stop_all_metrics() + if context_id: + await self.append_to_audio_context( + context_id, TTSStoppedFrame(context_id=context_id) + ) + await self.remove_audio_context(context_id) + elif msg_type == "error": + await self.stop_all_metrics() + error_detail = msg.get("message") or msg.get("error") or str(msg) + if context_id: + await self.append_to_audio_context( + context_id, TTSStoppedFrame(context_id=context_id) + ) + await self.remove_audio_context(context_id) + await self.push_error(error_msg=f"xAI TTS error: {error_detail}") + else: + logger.debug(f"{self}: unhandled xAI message type: {msg_type}") + + @traced_tts + async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]: + """Generate TTS audio from text using xAI's streaming WebSocket API.""" + logger.debug(f"{self}: Generating TTS [{text}]") + + try: + if not self._websocket or self._websocket.state is State.CLOSED: + await self._connect() + + try: + await self._get_websocket().send(json.dumps({"type": "text.delta", "delta": text})) + await self.start_tts_usage_metrics(text) + except Exception as e: + yield ErrorFrame(error=f"Unknown error occurred: {e}") + yield TTSStoppedFrame(context_id=context_id) + await self._disconnect() + await self._connect() + return + yield None + except Exception as e: + yield ErrorFrame(error=f"Unknown error occurred: {e}") diff --git a/src/pipecat/turns/user_start/__init__.py b/src/pipecat/turns/user_start/__init__.py index 94d12708d..14de5d28b 100644 --- a/src/pipecat/turns/user_start/__init__.py +++ b/src/pipecat/turns/user_start/__init__.py @@ -11,9 +11,15 @@ from .transcription_user_turn_start_strategy import TranscriptionUserTurnStartSt from .vad_user_turn_start_strategy import VADUserTurnStartStrategy from .wake_phrase_user_turn_start_strategy import WakePhraseUserTurnStartStrategy +try: + from .krisp_viva_ip_user_turn_start_strategy import KrispVivaIPUserTurnStartStrategy +except ImportError: + KrispVivaIPUserTurnStartStrategy = None + __all__ = [ "BaseUserTurnStartStrategy", "ExternalUserTurnStartStrategy", + "KrispVivaIPUserTurnStartStrategy", "MinWordsUserTurnStartStrategy", "TranscriptionUserTurnStartStrategy", "UserTurnStartedParams", diff --git a/src/pipecat/turns/user_start/base_user_turn_start_strategy.py b/src/pipecat/turns/user_start/base_user_turn_start_strategy.py index 401178cc7..b424d7a9e 100644 --- a/src/pipecat/turns/user_start/base_user_turn_start_strategy.py +++ b/src/pipecat/turns/user_start/base_user_turn_start_strategy.py @@ -101,7 +101,7 @@ class BaseUserTurnStartStrategy(BaseObject): """Reset the strategy to its initial state.""" pass - async def process_frame(self, frame: Frame) -> ProcessFrameResult: + async def process_frame(self, frame: Frame) -> ProcessFrameResult | None: """Process an incoming frame. Subclasses should override this to implement logic that decides whether @@ -111,8 +111,8 @@ class BaseUserTurnStartStrategy(BaseObject): frame: The frame to be processed. Returns: - A ProcessFrameResult indicating the outcome. Subclasses that return - None are treated as CONTINUE for backward compatibility. + A ProcessFrameResult indicating the outcome, or None (treated as + CONTINUE for backward compatibility). """ pass diff --git a/src/pipecat/turns/user_start/krisp_viva_ip_user_turn_start_strategy.py b/src/pipecat/turns/user_start/krisp_viva_ip_user_turn_start_strategy.py new file mode 100644 index 000000000..807bc8154 --- /dev/null +++ b/src/pipecat/turns/user_start/krisp_viva_ip_user_turn_start_strategy.py @@ -0,0 +1,282 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""User turn start strategy using Krisp Interruption Prediction (IP). + +This strategy uses Krisp's IP model to distinguish genuine user interruptions +from backchannels (e.g. "uh-huh", "yeah"). Instead of triggering a user turn +on every VAD speech event, it collects audio after VAD detects speech and runs +the IP model to predict whether the speech is a real interruption. + +Only when the IP model's probability exceeds the configured threshold is +``trigger_user_turn_started()`` called. This prevents the bot from being +interrupted by brief acknowledgements or filler words. +""" + +import os + +import numpy as np +from loguru import logger + +from pipecat.audio.krisp_instance import ( + KrispVivaSDKManager, + int_to_krisp_frame_duration, + int_to_krisp_sample_rate, +) +from pipecat.frames.frames import ( + BotStoppedSpeakingFrame, + Frame, + InputAudioRawFrame, + VADUserStartedSpeakingFrame, + VADUserStoppedSpeakingFrame, +) +from pipecat.turns.types import ProcessFrameResult +from pipecat.turns.user_start.base_user_turn_start_strategy import BaseUserTurnStartStrategy + +try: + import krisp_audio +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error( + "In order to use KrispVivaIPUserTurnStartStrategy, you need to install krisp_audio." + ) + raise Exception(f"Missing module: {e}") + + +class KrispVivaIPUserTurnStartStrategy(BaseUserTurnStartStrategy): + """User turn start strategy using Krisp VIVA Interruption Prediction. + + When VAD detects user speech, this strategy feeds audio frames into + the Krisp VIVA IP model. The model outputs a probability indicating + whether the speech is a genuine interruption (as opposed to a + backchannel). A user turn is triggered only when this probability + exceeds the configured threshold. + + This strategy is designed to work alongside other start strategies + (e.g. ``TranscriptionUserTurnStartStrategy`` as a fallback) via the + strategy list in ``UserTurnStrategies``. + + Example:: + + from pipecat.turns.user_start import KrispVivaIPUserTurnStartStrategy + + strategies = UserTurnStrategies( + start=[ + KrispVivaIPUserTurnStartStrategy( + model_path="/path/to/ip_model.kef", + threshold=0.5, + ), + TranscriptionUserTurnStartStrategy(), + ], + ) + """ + + def __init__( + self, + *, + model_path: str | None = None, + threshold: float = 0.5, + frame_duration_ms: int = 20, + api_key: str = "", + **kwargs, + ): + """Initialize the Krisp VIVA IP user turn start strategy. + + Args: + model_path: Path to the Krisp VIVA IP model file (.kef). If None, + uses the KRISP_VIVA_IP_MODEL_PATH environment variable. + threshold: IP probability threshold (0.0 to 1.0). When the model's + output exceeds this value, the speech is classified as a genuine + interruption. + frame_duration_ms: Frame duration in milliseconds for IP processing. + Supported values: 10, 15, 20, 30, 32. + api_key: Krisp SDK API key. If empty, falls back to the + KRISP_VIVA_API_KEY environment variable. + **kwargs: Additional arguments passed to BaseUserTurnStartStrategy. + """ + super().__init__(**kwargs) + + self._threshold = threshold + self._frame_duration_ms = frame_duration_ms + self._api_key = api_key + + self._model_path = model_path or os.getenv("KRISP_VIVA_IP_MODEL_PATH") + if not self._model_path: + raise ValueError( + "IP model path must be provided via model_path or " + "KRISP_VIVA_IP_MODEL_PATH environment variable." + ) + if not self._model_path.endswith(".kef"): + raise ValueError("Model is expected with .kef extension") + if not os.path.isfile(self._model_path): + raise FileNotFoundError(f"IP model file not found: {self._model_path}") + + self._sdk_acquired = False + self._ip_session = None + self._samples_per_frame: int | None = None + self._sample_rate: int | None = None + + # State tracking + self._speech_active = False + self._audio_buffer = bytearray() + self._decision_made = False + + # Acquire SDK + try: + KrispVivaSDKManager.acquire(api_key=api_key) + self._sdk_acquired = True + except Exception as e: + raise RuntimeError(f"Failed to initialize Krisp SDK: {e}") + + async def cleanup(self): + """Release Krisp SDK resources.""" + if self._sdk_acquired: + try: + self._ip_session = None + KrispVivaSDKManager.release() + self._sdk_acquired = False + except Exception as e: + logger.error(f"Error cleaning up Krisp VIVA IP strategy: {e}", exc_info=True) + + def _ensure_session(self, sample_rate: int): + """Create or re-create the IP session when sample rate changes. + + Args: + sample_rate: Audio sample rate in Hz. + """ + if self._sample_rate == sample_rate and self._ip_session is not None: + return + + self._sample_rate = sample_rate + self._samples_per_frame = int((sample_rate * self._frame_duration_ms) / 1000) + + model_info = krisp_audio.ModelInfo() + model_info.path = self._model_path + + ip_cfg = krisp_audio.IpSessionConfig() + ip_cfg.inputSampleRate = int_to_krisp_sample_rate(sample_rate) + ip_cfg.inputFrameDuration = int_to_krisp_frame_duration(self._frame_duration_ms) + ip_cfg.modelInfo = model_info + + self._ip_session = krisp_audio.IpFloat.create(ip_cfg) + logger.debug(f"Krisp VIVA IP session created (sample_rate={sample_rate})") + + def _reset_state(self): + """Reset speech tracking state for the next candidate interruption.""" + self._speech_active = False + self._audio_buffer.clear() + self._decision_made = False + + async def reset(self): + """Reset the strategy to its initial state.""" + await super().reset() + self._reset_state() + + async def process_frame(self, frame: Frame) -> ProcessFrameResult: + """Process a frame to detect genuine user interruptions. + + On ``VADUserStartedSpeakingFrame``, begins collecting audio. + On ``InputAudioRawFrame``, feeds audio through the IP model and + triggers a user turn if the interruption probability exceeds the + threshold. + On ``VADUserStoppedSpeakingFrame`` or ``BotStoppedSpeakingFrame``, + resets the candidate state. + + Args: + frame: The incoming frame. + + Returns: + STOP if a genuine interruption was detected, CONTINUE otherwise. + """ + if isinstance(frame, VADUserStartedSpeakingFrame): + return await self._handle_vad_started(frame) + elif isinstance(frame, InputAudioRawFrame): + return await self._handle_audio(frame) + elif isinstance(frame, (VADUserStoppedSpeakingFrame, BotStoppedSpeakingFrame)): + return await self._handle_reset(frame) + + return ProcessFrameResult.CONTINUE + + async def _handle_vad_started(self, frame: VADUserStartedSpeakingFrame) -> ProcessFrameResult: + """Begin collecting audio for interruption classification. + + Args: + frame: The VAD speech-start frame. + + Returns: + Always CONTINUE; the decision is deferred until enough audio is processed. + """ + logger.trace("Krisp VIVA IP: VAD speech started, collecting audio for classification") + self._speech_active = True + self._audio_buffer.clear() + self._decision_made = False + return ProcessFrameResult.CONTINUE + + async def _handle_audio(self, frame: InputAudioRawFrame) -> ProcessFrameResult: + """Feed audio to the IP model and check for genuine interruption. + + Args: + frame: Raw audio input frame. + + Returns: + STOP if the model detects a genuine interruption, CONTINUE otherwise. + """ + if not self._speech_active or self._decision_made: + return ProcessFrameResult.CONTINUE + + self._ensure_session(frame.sample_rate) + + if self._ip_session is None or self._samples_per_frame is None: + logger.warning("IP session not ready, skipping frame") + return ProcessFrameResult.CONTINUE + + self._audio_buffer.extend(frame.audio) + + total_samples = len(self._audio_buffer) // 2 # 2 bytes per int16 sample + num_complete_frames = total_samples // self._samples_per_frame + + if num_complete_frames == 0: + return ProcessFrameResult.CONTINUE + + complete_samples_count = num_complete_frames * self._samples_per_frame + bytes_to_process = complete_samples_count * 2 + + audio_to_process = bytes(self._audio_buffer[:bytes_to_process]) + self._audio_buffer = self._audio_buffer[bytes_to_process:] + + audio_int16 = np.frombuffer(audio_to_process, dtype=np.int16) + audio_float32 = audio_int16.astype(np.float32) / 32768.0 + frames = audio_float32.reshape(-1, self._samples_per_frame) + + for ip_frame in frames: + ip_prob = self._ip_session.process(ip_frame.tolist(), self._speech_active) + + if ip_prob >= self._threshold: + logger.debug( + f"Krisp VIVA IP: genuine interruption detected (prob={ip_prob:.3f}, " + f"threshold={self._threshold})" + ) + self._decision_made = True + await self.trigger_user_turn_started() + return ProcessFrameResult.STOP + + return ProcessFrameResult.CONTINUE + + async def _handle_reset( + self, frame: VADUserStoppedSpeakingFrame | BotStoppedSpeakingFrame + ) -> ProcessFrameResult: + """Reset state when the candidate interruption window ends. + + Args: + frame: The frame signaling end of speech or bot output. + + Returns: + Always CONTINUE. + """ + if self._speech_active: + logger.trace("Krisp VIVA IP: speech segment ended, resetting state") + self._reset_state() + return ProcessFrameResult.CONTINUE diff --git a/src/pipecat/turns/user_stop/base_user_turn_stop_strategy.py b/src/pipecat/turns/user_stop/base_user_turn_stop_strategy.py index 1f8497359..e5dc1bd66 100644 --- a/src/pipecat/turns/user_stop/base_user_turn_stop_strategy.py +++ b/src/pipecat/turns/user_stop/base_user_turn_stop_strategy.py @@ -89,7 +89,7 @@ class BaseUserTurnStopStrategy(BaseObject): """Reset the strategy to its initial state.""" pass - async def process_frame(self, frame: Frame) -> ProcessFrameResult: + async def process_frame(self, frame: Frame) -> ProcessFrameResult | None: """Process an incoming frame to decide whether the user stopped speaking. Subclasses should override this to implement logic that decides whether @@ -99,8 +99,8 @@ class BaseUserTurnStopStrategy(BaseObject): frame: The frame to be analyzed. Returns: - A ProcessFrameResult indicating the outcome. Subclasses that return - None are treated as CONTINUE for backward compatibility. + A ProcessFrameResult indicating the outcome, or None (treated as + CONTINUE for backward compatibility). """ pass diff --git a/src/pipecat/turns/user_stop/speech_timeout_user_turn_stop_strategy.py b/src/pipecat/turns/user_stop/speech_timeout_user_turn_stop_strategy.py index e37b55756..254fa60b1 100644 --- a/src/pipecat/turns/user_stop/speech_timeout_user_turn_stop_strategy.py +++ b/src/pipecat/turns/user_stop/speech_timeout_user_turn_stop_strategy.py @@ -7,7 +7,6 @@ """Speech timeout-based user turn stop strategy.""" import asyncio -import time from loguru import logger @@ -25,20 +24,25 @@ from pipecat.utils.asyncio.task_manager import BaseTaskManager class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): - """User turn stop strategy that uses a configurable timeout to determine if the user is done speaking. + """User turn stop strategy using two independent timers after VAD stop. - After the user stops speaking (detected by VAD), this strategy waits for a - configurable timeout before triggering the end of the user's turn. The - timeout accounts for two factors: + After the user stops speaking (detected by VAD), this strategy runs two + independent timers. The user turn stop is triggered only when both have + finished and at least one transcript has been received: - - user_speech_timeout: Time to wait for the user to potentially say more - after they pause. - - stt_timeout: The P99 time for the STT service to return a transcription - after the user stops speaking, adjusted by the VAD stop_secs. + - user_speech_timeout: Policy floor — the window in which the user may + resume speaking after a pause. Always runs to completion. + - stt_timeout: Safety net for STT latency — the P99 time for the STT + service to return a final transcript after VAD stop, adjusted by the + VAD stop_secs. Short-circuited when the STT service emits a finalized + transcript (TranscriptionFrame.finalized=True), since finalization + means STT has nothing more to send. - For services that support finalization (TranscriptionFrame.finalized=True), - the turn can be triggered immediately once the finalized transcript is - received and the user resume speaking timeout has elapsed. + Fallback: when a transcript arrives without a VAD stop event, the + user_speech_timeout timer measures inactivity since the last transcript + (rearmed on each transcript). stt_timeout has no meaning here since it + is defined relative to VAD stop, and STT has already emitted a + transcript — so the stt wait is marked done immediately. """ def __init__(self, *, user_speech_timeout: float = 0.6, **kwargs): @@ -59,8 +63,11 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): self._vad_user_speaking = False self._transcript_finalized = False self._vad_stopped_time: float | None = None - self._timeout_task: asyncio.Task | None = None - self._timeout_expired: bool = False + + self._user_speech_timeout_task: asyncio.Task | None = None + self._stt_timeout_task: asyncio.Task | None = None + self._user_speech_wait_done: bool = False + self._stt_wait_done: bool = False async def reset(self): """Reset the strategy to its initial state.""" @@ -69,10 +76,9 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): self._vad_user_speaking = False self._transcript_finalized = False self._vad_stopped_time = None - self._timeout_expired = False - if self._timeout_task: - await self.task_manager.cancel_task(self._timeout_task) - self._timeout_task = None + self._user_speech_wait_done = False + self._stt_wait_done = False + await self._cancel_all_tasks() async def setup(self, task_manager: BaseTaskManager): """Initialize the strategy with the given task manager. @@ -85,9 +91,7 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): async def cleanup(self): """Cleanup the strategy.""" await super().cleanup() - if self._timeout_task: - await self.task_manager.cancel_task(self._timeout_task) - self._timeout_task = None + await self._cancel_all_tasks() async def process_frame(self, frame: Frame) -> ProcessFrameResult: """Process an incoming frame to update strategy state. @@ -105,8 +109,10 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): self._stt_timeout = frame.ttfs_p99_latency self._stop_secs_warned = False elif isinstance(frame, VADUserStartedSpeakingFrame): + logger.debug(f"{self} VADUserStartedSpeakingFrame received") await self._handle_vad_user_started_speaking(frame) elif isinstance(frame, VADUserStoppedSpeakingFrame): + logger.debug(f"{self} VADUserStoppedSpeakingFrame received") await self._handle_vad_user_stopped_speaking(frame) elif isinstance(frame, TranscriptionFrame): await self._handle_transcription(frame) @@ -118,11 +124,9 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): self._vad_user_speaking = True self._transcript_finalized = False self._vad_stopped_time = None - self._timeout_expired = False - # Cancel any pending timeout - if self._timeout_task: - await self.task_manager.cancel_task(self._timeout_task) - self._timeout_task = None + self._user_speech_wait_done = False + self._stt_wait_done = False + await self._cancel_all_tasks() async def _handle_vad_user_stopped_speaking(self, frame: VADUserStoppedSpeakingFrame): """Handle when the VAD indicates the user has stopped speaking.""" @@ -150,59 +154,69 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): f"user_turn_stop_timeout parameter in the LLMUserAggregatorParams." ) - # Start the timeout task - timeout = self._calculate_timeout() - self._timeout_task = self.task_manager.create_task( - self._timeout_handler(timeout), f"{self}::_timeout_handler" - ) - # Make sure the task is scheduled. + # user_speech_timeout is the policy floor and always runs. A prior + # fallback-mode run of the same timer is superseded here. + await self._restart_user_speech_timer() + + # stt_timeout is a safety net. Short-circuit it if the transcript is + # already finalized, or if the VAD stop_secs already covered it. + self._stt_wait_done = False + effective_stt_wait = max(0.0, self._stt_timeout - self._stop_secs) + if self._transcript_finalized or effective_stt_wait <= 0: + self._stt_wait_done = True + else: + self._stt_timeout_task = self.task_manager.create_task( + self._stt_timeout_handler(effective_stt_wait), + f"{self}::_stt_timeout_handler", + ) + + # Make sure the tasks are scheduled. await asyncio.sleep(0) async def _handle_transcription(self, frame: TranscriptionFrame): """Handle user transcription.""" self._text += frame.text + if frame.finalized: self._transcript_finalized = True - # For finalized transcripts, check if we can trigger early + # Short-circuit the stt_timeout safety net: STT has told us + # there's nothing more coming. + if not self._stt_wait_done: + self._stt_wait_done = True + if self._stt_timeout_task: + await self.task_manager.cancel_task(self._stt_timeout_task) + self._stt_timeout_task = None + + # If both waits are already done, the turn was waiting on text — + # trigger now. + if self._user_speech_wait_done and self._stt_wait_done: await self._maybe_trigger_user_turn_stopped() - elif self._timeout_expired: - # The p99 timeout already elapsed without a transcript. Now that - # we have one, trigger the turn stop immediately. - await self.trigger_user_turn_stopped() return - # Fallback: handle transcripts when no VAD stop was received. - # This handles edge cases where transcripts arrive without VAD firing. - # _vad_stopped_time is None means VAD stopped hasn't been received yet. - # In fallback mode, reset timeout on each transcript to wait for inactivity. + # Fallback: transcript arrived without a VAD stop. Measure inactivity + # since the last transcript with the user_speech_timer. stt_timeout + # has no meaning here (it's defined relative to VAD stop), so mark + # the stt wait done immediately. if not self._vad_user_speaking and self._vad_stopped_time is None: - # Cancel existing fallback timeout if any - if self._timeout_task: - await self.task_manager.cancel_task(self._timeout_task) - timeout = self._calculate_timeout() - self._timeout_task = self.task_manager.create_task( - self._timeout_handler(timeout), f"{self}::_timeout_handler" - ) - # Make sure the task is scheduled. - await asyncio.sleep(0) + self._stt_wait_done = True + await self._restart_user_speech_timer() - def _calculate_timeout(self) -> float: - """Calculate the timeout value based on current state. + async def _restart_user_speech_timer(self): + """Cancel any running user_speech timer and start a fresh one.""" + if self._user_speech_timeout_task: + await self.task_manager.cancel_task(self._user_speech_timeout_task) + self._user_speech_timeout_task = None + self._user_speech_wait_done = False + self._user_speech_timeout_task = self.task_manager.create_task( + self._user_speech_timeout_handler(self._user_speech_timeout), + f"{self}::_user_speech_timeout_handler", + ) + # Make sure the task is scheduled so it can't be cancelled before + # starting (which would leave its coroutine un-awaited). + await asyncio.sleep(0) - Returns: - The timeout in seconds to wait after VAD stopped speaking. - """ - # Adjust STT timeout by VAD stop_secs since that time has already elapsed - effective_stt_wait = max(0, self._stt_timeout - self._stop_secs) - - # If transcript is already finalized, we don't need to wait for STT - if self._transcript_finalized: - return self._user_speech_timeout - - return max(effective_stt_wait, self._user_speech_timeout) - - async def _timeout_handler(self, timeout: float): - """Wait for the timeout then trigger user turn stopped if conditions met. + async def _user_speech_timeout_handler(self, timeout: float): + """Wait user_speech_timeout then attempt to trigger user turn stopped. Args: timeout: The timeout in seconds to wait. @@ -212,36 +226,46 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy): except asyncio.CancelledError: return finally: - self._timeout_task = None + self._user_speech_timeout_task = None - self._timeout_expired = True + self._user_speech_wait_done = True + await self._maybe_trigger_user_turn_stopped() + + async def _stt_timeout_handler(self, timeout: float): + """Wait stt_timeout then attempt to trigger user turn stopped. + + Args: + timeout: The timeout in seconds to wait. + """ + try: + await asyncio.sleep(timeout) + except asyncio.CancelledError: + return + finally: + self._stt_timeout_task = None + + self._stt_wait_done = True await self._maybe_trigger_user_turn_stopped() async def _maybe_trigger_user_turn_stopped(self): - """Trigger user turn stopped if conditions are met. + """Trigger user turn stopped if all required conditions are met. - Conditions: - - User is not currently speaking - - We have transcription text - - Either the timeout has elapsed OR we have a finalized transcript - and user_speech_timeout has elapsed + Both timers must be done (stt is marked done immediately on the + fallback path and when finalization short-circuits the safety net), + the user must not be currently speaking, and at least one transcript + must have been received. """ if self._vad_user_speaking or not self._text: return - # For finalized transcripts, check if user_speech_timeout has elapsed. - # If elapsed, trigger user turn stopped immediately. Else, wait for user resume - # speaking timeout. - if self._transcript_finalized and self._vad_stopped_time is not None: - elapsed = time.time() - self._vad_stopped_time - if elapsed >= self._user_speech_timeout: - # Cancel any remaining timeout since we're triggering now - if self._timeout_task: - await self.task_manager.cancel_task(self._timeout_task) - self._timeout_task = None - await self.trigger_user_turn_stopped() - return - - # For non-finalized, only trigger if timeout task has completed - if self._timeout_task is None: + if self._user_speech_wait_done and self._stt_wait_done: await self.trigger_user_turn_stopped() + + async def _cancel_all_tasks(self): + """Cancel any running timer tasks and clear the handles.""" + if self._user_speech_timeout_task: + await self.task_manager.cancel_task(self._user_speech_timeout_task) + self._user_speech_timeout_task = None + if self._stt_timeout_task: + await self.task_manager.cancel_task(self._stt_timeout_task) + self._stt_timeout_task = None diff --git a/src/pipecat/turns/user_turn_completion_mixin.py b/src/pipecat/turns/user_turn_completion_mixin.py index 1d8da1e07..51d6c7828 100644 --- a/src/pipecat/turns/user_turn_completion_mixin.py +++ b/src/pipecat/turns/user_turn_completion_mixin.py @@ -26,7 +26,7 @@ from pipecat.frames.frames import ( LLMRunFrame, LLMTextFrame, ) -from pipecat.processors.frame_processor import FrameDirection +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor # Turn completion markers USER_TURN_COMPLETE_MARKER = "✓" @@ -178,7 +178,7 @@ class UserTurnCompletionConfig: return self.incomplete_long_prompt or DEFAULT_INCOMPLETE_LONG_PROMPT -class UserTurnCompletionLLMServiceMixin: +class UserTurnCompletionLLMServiceMixin(FrameProcessor): """Mixin that adds turn completion detection to LLM services. This mixin provides methods to push LLM text with turn completion detection. @@ -292,7 +292,7 @@ class UserTurnCompletionLLMServiceMixin: # Push through pipeline to trigger LLM response await self.push_frame( - LLMMessagesAppendFrame(messages=[{"role": "system", "content": prompt}]) + LLMMessagesAppendFrame(messages=[{"role": "developer", "content": prompt}]) ) await self.push_frame(LLMRunFrame()) diff --git a/src/pipecat/utils/context/llm_context_summarization.py b/src/pipecat/utils/context/llm_context_summarization.py index 85a7e6e19..59de6f4fa 100644 --- a/src/pipecat/utils/context/llm_context_summarization.py +++ b/src/pipecat/utils/context/llm_context_summarization.py @@ -20,7 +20,11 @@ if TYPE_CHECKING: from loguru import logger -from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage +from pipecat.processors.aggregators.llm_context import ( + LLMContext, + LLMContextMessage, + LLMSpecificMessage, +) # Fallback timeout (seconds) used when summarization_timeout is None. DEFAULT_SUMMARIZATION_TIMEOUT = 120.0 @@ -269,7 +273,7 @@ class LLMMessagesToSummarize: last_summarized_index: Index of the last message being summarized """ - messages: list[dict] + messages: list[LLMContextMessage] last_summarized_index: int @@ -415,7 +419,7 @@ class LLMContextSummarizationUtil: @staticmethod def _get_earliest_function_call_not_resolved_in_range( - messages: list[dict], start_idx: int, summary_end: int + messages: list[LLMContextMessage], start_idx: int, summary_end: int ) -> int: """Find the earliest message index with incomplete function calls. @@ -470,9 +474,10 @@ class LLMContextSummarizationUtil: if role == "tool": tool_call_id = msg.get("tool_call_id") if tool_call_id and tool_call_id in pending_tool_calls: - if not LLMContextSummarizationUtil._is_tool_message_pending( - msg.get("content", "") - ): + content = msg.get("content", "") + if not isinstance(content, str): + content = "" + if not LLMContextSummarizationUtil._is_tool_message_pending(content): pending_tool_calls.pop(tool_call_id) # Check for async tool completion — a developer message with @@ -480,7 +485,10 @@ class LLMContextSummarizationUtil: # async result has arrived and the call is now resolved. if role == "developer": try: - parsed = json.loads(msg.get("content", "")) + content = msg.get("content", "") + if not isinstance(content, str): + continue + parsed = json.loads(content) if ( isinstance(parsed, dict) and parsed.get("type") == "async_tool" diff --git a/src/pipecat/utils/frame_queue.py b/src/pipecat/utils/frame_queue.py index 1b787362d..14347681a 100644 --- a/src/pipecat/utils/frame_queue.py +++ b/src/pipecat/utils/frame_queue.py @@ -58,7 +58,7 @@ class FrameQueue(asyncio.Queue): Returns: True if at least one enqueued frame is an instance of ``frame_type``. """ - for item in self._queue: + for item in self._queue: # pyright: ignore[reportAttributeAccessIssue] if isinstance(self._frame_getter(item), frame_type): return True return False diff --git a/src/pipecat/utils/string.py b/src/pipecat/utils/string.py index 55b1c2d53..c464e189f 100644 --- a/src/pipecat/utils/string.py +++ b/src/pipecat/utils/string.py @@ -234,7 +234,7 @@ class TextPartForConcatenation: includes_inter_part_spaces: bool def __str__(self): - return f"{self.name}(text: [{self.text}], includes_inter_part_spaces: {self.includes_inter_part_spaces})" + return f"{type(self).__name__}(text: [{self.text}], includes_inter_part_spaces: {self.includes_inter_part_spaces})" def concatenate_aggregated_text(text_parts: list[TextPartForConcatenation]) -> str: diff --git a/src/pipecat/utils/text/base_text_aggregator.py b/src/pipecat/utils/text/base_text_aggregator.py index 99ca145b6..b1b933c81 100644 --- a/src/pipecat/utils/text/base_text_aggregator.py +++ b/src/pipecat/utils/text/base_text_aggregator.py @@ -125,7 +125,7 @@ class BaseTextAggregator(ABC): """ pass # Make this a generator to satisfy type checker - yield # pragma: no cover + yield # pyright: ignore[reportReturnType] # pragma: no cover @abstractmethod async def flush(self) -> Aggregation | None: diff --git a/src/pipecat/utils/text/pattern_pair_aggregator.py b/src/pipecat/utils/text/pattern_pair_aggregator.py index 975413c78..67e044152 100644 --- a/src/pipecat/utils/text/pattern_pair_aggregator.py +++ b/src/pipecat/utils/text/pattern_pair_aggregator.py @@ -273,7 +273,7 @@ class PatternPairAggregator(SimpleTextAggregator): # Which is why we base the return on the first found. if start_count > end_count: start_index = text.find(start) - return [start_index, pattern_info] + return (start_index, pattern_info) return None diff --git a/src/pipecat/utils/tracing/service_attributes.py b/src/pipecat/utils/tracing/service_attributes.py index d9b86a9a4..a8b33e225 100644 --- a/src/pipecat/utils/tracing/service_attributes.py +++ b/src/pipecat/utils/tracing/service_attributes.py @@ -440,7 +440,7 @@ def add_openai_realtime_span_attributes( if isinstance(tool, dict) and "name" in tool: tool_names.append(tool["name"]) elif hasattr(tool, "name"): - tool_names.append(tool.name) + tool_names.append(getattr(tool, "name")) elif isinstance(tool, dict) and "function" in tool and "name" in tool["function"]: tool_names.append(tool["function"]["name"]) @@ -455,7 +455,7 @@ def add_openai_realtime_span_attributes( if function_calls: call = function_calls[0] if hasattr(call, "name"): - span.set_attribute("function_calls.first_name", call.name) + span.set_attribute("function_calls.first_name", getattr(call, "name")) elif isinstance(call, dict) and "name" in call: span.set_attribute("function_calls.first_name", call["name"]) diff --git a/tests/test_krisp_ip_user_turn_start_strategy.py b/tests/test_krisp_ip_user_turn_start_strategy.py new file mode 100644 index 000000000..bb34d879a --- /dev/null +++ b/tests/test_krisp_ip_user_turn_start_strategy.py @@ -0,0 +1,236 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os +import sys +import tempfile +import unittest +from unittest.mock import MagicMock, patch + +import numpy as np + +# Mock package version check before importing pipecat (development mode) +_version_patcher = patch("importlib.metadata.version", return_value="0.0.0-dev") +_version_patcher.start() + +# Mock krisp_audio before any pipecat import that loads krisp_instance / VIVA IP strategy +mock_krisp_audio = MagicMock() +mock_krisp_audio.SamplingRate.Sr8000Hz = 8000 +mock_krisp_audio.SamplingRate.Sr16000Hz = 16000 +mock_krisp_audio.SamplingRate.Sr24000Hz = 24000 +mock_krisp_audio.SamplingRate.Sr32000Hz = 32000 +mock_krisp_audio.SamplingRate.Sr44100Hz = 44100 +mock_krisp_audio.SamplingRate.Sr48000Hz = 48000 +mock_krisp_audio.FrameDuration.Fd10ms = "10ms" +mock_krisp_audio.FrameDuration.Fd15ms = "15ms" +mock_krisp_audio.FrameDuration.Fd20ms = "20ms" +mock_krisp_audio.FrameDuration.Fd30ms = "30ms" +mock_krisp_audio.FrameDuration.Fd32ms = "32ms" + +sys.modules["krisp_audio"] = mock_krisp_audio + +mock_pipecat_krisp = MagicMock() +sys.modules["pipecat_ai_krisp"] = mock_pipecat_krisp +sys.modules["pipecat_ai_krisp.audio"] = MagicMock() +sys.modules["pipecat_ai_krisp.audio.krisp_processor"] = MagicMock() + +from pipecat.frames.frames import ( + BotStartedSpeakingFrame, + BotStoppedSpeakingFrame, + InputAudioRawFrame, + TranscriptionFrame, + VADUserStartedSpeakingFrame, + VADUserStoppedSpeakingFrame, +) +from pipecat.turns.types import ProcessFrameResult +from pipecat.turns.user_start.krisp_viva_ip_user_turn_start_strategy import ( + KrispVivaIPUserTurnStartStrategy, +) + +STRATEGY_MODULE = "pipecat.turns.user_start.krisp_viva_ip_user_turn_start_strategy" + + +def _int16_silence(num_samples: int) -> bytes: + return np.zeros(num_samples, dtype=np.int16).tobytes() + + +class TestKrispVivaIPUserTurnStartStrategy(unittest.IsolatedAsyncioTestCase): + """Tests for KrispVivaIPUserTurnStartStrategy with mocked krisp_audio.""" + + def setUp(self): + self.temp_model_file = tempfile.NamedTemporaryFile(suffix=".kef", delete=False) + self.temp_model_file.write(b"dummy") + self.temp_model_file.close() + self.model_path = self.temp_model_file.name + + self.mock_krisp_audio = mock_krisp_audio + self.mock_krisp_audio.reset_mock() + self.mock_krisp_audio.ModelInfo.reset_mock() + self.mock_krisp_audio.IpSessionConfig.reset_mock() + self.mock_krisp_audio.IpFloat.reset_mock() + + self.mock_model_info = MagicMock() + self.mock_krisp_audio.ModelInfo.return_value = self.mock_model_info + + self.mock_ip_cfg = MagicMock() + self.mock_krisp_audio.IpSessionConfig.return_value = self.mock_ip_cfg + + self.mock_ip_session = MagicMock() + self.mock_krisp_audio.IpFloat.create.return_value = self.mock_ip_session + + self.krisp_patch = patch(f"{STRATEGY_MODULE}.krisp_audio", self.mock_krisp_audio) + self.krisp_patch.start() + + self.sdk_patcher = patch(f"{STRATEGY_MODULE}.KrispVivaSDKManager") + self.mock_sdk_manager = self.sdk_patcher.start() + self.mock_sdk_manager.acquire = MagicMock() + self.mock_sdk_manager.release = MagicMock() + + def tearDown(self): + self.krisp_patch.stop() + self.sdk_patcher.stop() + if os.path.exists(self.model_path): + os.unlink(self.model_path) + + def _make_strategy(self, *, threshold: float = 0.5, frame_duration_ms: int = 20): + return KrispVivaIPUserTurnStartStrategy( + model_path=self.model_path, + threshold=threshold, + frame_duration_ms=frame_duration_ms, + api_key="test-key", + ) + + def _audio_frame(self, sample_rate: int = 16000, frame_duration_ms: int = 20): + samples = int(sample_rate * frame_duration_ms / 1000) + return InputAudioRawFrame( + audio=_int16_silence(samples), + sample_rate=sample_rate, + num_channels=1, + ) + + async def test_interruption_detected_emits_turn_and_stop(self): + self.mock_ip_session.process = MagicMock(return_value=0.87) + + strategy = self._make_strategy(threshold=0.5) + try: + fired = False + + @strategy.event_handler("on_user_turn_started") + async def on_user_turn_started(strategy, params): + nonlocal fired + fired = True + + await strategy.process_frame(VADUserStartedSpeakingFrame()) + result = await strategy.process_frame(self._audio_frame()) + + self.assertTrue(fired) + self.assertEqual(result, ProcessFrameResult.STOP) + self.mock_ip_session.process.assert_called() + finally: + await strategy.cleanup() + + async def test_backchannel_suppressed_no_event_continue(self): + self.mock_ip_session.process = MagicMock(return_value=0.23) + + strategy = self._make_strategy(threshold=0.5) + try: + fired = False + + @strategy.event_handler("on_user_turn_started") + async def on_user_turn_started(strategy, params): + nonlocal fired + fired = True + + await strategy.process_frame(VADUserStartedSpeakingFrame()) + result = await strategy.process_frame(self._audio_frame()) + + self.assertFalse(fired) + self.assertEqual(result, ProcessFrameResult.CONTINUE) + finally: + await strategy.cleanup() + + async def test_reset_on_vad_stopped_clears_state(self): + self.mock_ip_session.process = MagicMock(return_value=0.1) + + strategy = self._make_strategy(threshold=0.5) + try: + await strategy.process_frame(VADUserStartedSpeakingFrame()) + await strategy.process_frame(self._audio_frame()) + self.mock_ip_session.process.reset_mock() + + await strategy.process_frame(VADUserStoppedSpeakingFrame()) + result = await strategy.process_frame(self._audio_frame()) + + self.assertEqual(result, ProcessFrameResult.CONTINUE) + self.mock_ip_session.process.assert_not_called() + finally: + await strategy.cleanup() + + async def test_reset_on_bot_stopped_clears_state(self): + self.mock_ip_session.process = MagicMock(return_value=0.1) + + strategy = self._make_strategy(threshold=0.5) + try: + await strategy.process_frame(VADUserStartedSpeakingFrame()) + await strategy.process_frame(self._audio_frame()) + self.mock_ip_session.process.reset_mock() + + await strategy.process_frame(BotStoppedSpeakingFrame()) + result = await strategy.process_frame(self._audio_frame()) + + self.assertEqual(result, ProcessFrameResult.CONTINUE) + self.mock_ip_session.process.assert_not_called() + finally: + await strategy.cleanup() + + async def test_no_op_before_vad_start(self): + self.mock_ip_session.process = MagicMock(return_value=0.99) + + strategy = self._make_strategy() + try: + result = await strategy.process_frame(self._audio_frame()) + self.assertEqual(result, ProcessFrameResult.CONTINUE) + self.mock_ip_session.process.assert_not_called() + finally: + await strategy.cleanup() + + async def test_decision_sticks_no_double_trigger(self): + self.mock_ip_session.process = MagicMock(return_value=0.9) + + strategy = self._make_strategy(threshold=0.5) + try: + count = 0 + + @strategy.event_handler("on_user_turn_started") + async def on_user_turn_started(strategy, params): + nonlocal count + count += 1 + + await strategy.process_frame(VADUserStartedSpeakingFrame()) + r1 = await strategy.process_frame(self._audio_frame()) + r2 = await strategy.process_frame(self._audio_frame()) + + self.assertEqual(r1, ProcessFrameResult.STOP) + self.assertEqual(r2, ProcessFrameResult.CONTINUE) + self.assertEqual(count, 1) + finally: + await strategy.cleanup() + + async def test_unrelated_frames_continue(self): + strategy = self._make_strategy() + try: + r1 = await strategy.process_frame(BotStartedSpeakingFrame()) + r2 = await strategy.process_frame( + TranscriptionFrame(text="hi", user_id="", timestamp="") + ) + self.assertEqual(r1, ProcessFrameResult.CONTINUE) + self.assertEqual(r2, ProcessFrameResult.CONTINUE) + finally: + await strategy.cleanup() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_krisp_sdk_manager.py b/tests/test_krisp_sdk_manager.py index 78a4d955f..2edbf4598 100644 --- a/tests/test_krisp_sdk_manager.py +++ b/tests/test_krisp_sdk_manager.py @@ -62,8 +62,15 @@ class TestKrispVivaSDKManager: def setup_method(self): """Reset mocks and SDK state before each test.""" mock_krisp_audio.reset_mock() + mock_krisp_audio.globalInit.side_effect = None mock_krisp_audio.getVersion.return_value = mock_version + # Ensure krisp_instance module uses THIS test's mock, not a stale + # reference cached from a different test file's sys.modules entry. + import pipecat.audio.krisp_instance as _ki + + _ki.krisp_audio = mock_krisp_audio + # Reset the SDK manager state for clean tests # We access internal state to ensure tests are isolated with KrispVivaSDKManager._lock: diff --git a/tests/test_user_turn_stop_strategy.py b/tests/test_user_turn_stop_strategy.py index ecaf5317f..9acceaf0f 100644 --- a/tests/test_user_turn_stop_strategy.py +++ b/tests/test_user_turn_stop_strategy.py @@ -529,6 +529,107 @@ class TestSpeechTimeoutUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase): # Non-finalized transcript received after timeout, triggers immediately self.assertTrue(should_start) + async def test_finalized_short_circuits_stt_wait(self): + """Finalized transcript cancels the stt_timeout safety net. + + user_speech_timeout still runs to completion as a policy floor, + but stt_timeout is skipped once STT says it's done. Net effect: + the turn stops at user_speech_timeout, not stt_timeout. + """ + stt_timeout = AGGREGATION_TIMEOUT * 4 + strategy = SpeechTimeoutUserTurnStopStrategy(user_speech_timeout=AGGREGATION_TIMEOUT) + await strategy.setup(self.task_manager) + await strategy.process_frame( + STTMetadataFrame(service_name="test", ttfs_p99_latency=stt_timeout) + ) + + should_start = None + + @strategy.event_handler("on_user_turn_stopped") + async def on_user_turn_stopped(strategy, params): + nonlocal should_start + should_start = True + + # S → E: starts user_speech_timeout (short) and stt_timeout (long). + await strategy.process_frame(VADUserStartedSpeakingFrame()) + await strategy.process_frame(VADUserStoppedSpeakingFrame()) + + # Finalized transcript arrives before user_speech_timeout elapses. + await strategy.process_frame( + TranscriptionFrame(text="Hello!", user_id="cat", timestamp="", finalized=True) + ) + # user_speech_timeout is still running, so no trigger yet. + self.assertIsNone(should_start) + + # user_speech_timeout elapses — stt_timeout was short-circuited, + # so the turn stops now rather than waiting for stt_timeout. + await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1) + self.assertTrue(should_start) + + async def test_non_finalized_waits_full_stt_timeout(self): + """Non-finalized transcript does not short-circuit stt_timeout. + + When STT never signals finalization, the stt_timeout safety net + must run its full course — the turn should not stop until the + longer of the two timers has elapsed. + """ + stt_timeout = AGGREGATION_TIMEOUT * 4 + strategy = SpeechTimeoutUserTurnStopStrategy(user_speech_timeout=AGGREGATION_TIMEOUT) + await strategy.setup(self.task_manager) + await strategy.process_frame( + STTMetadataFrame(service_name="test", ttfs_p99_latency=stt_timeout) + ) + + should_start = None + + @strategy.event_handler("on_user_turn_stopped") + async def on_user_turn_stopped(strategy, params): + nonlocal should_start + should_start = True + + # S → E: both timers start. + await strategy.process_frame(VADUserStartedSpeakingFrame()) + await strategy.process_frame(VADUserStoppedSpeakingFrame()) + + # Non-finalized transcript during the wait. + await strategy.process_frame(TranscriptionFrame(text="Hello!", user_id="cat", timestamp="")) + + # user_speech_timeout elapses but stt_timeout has not — no trigger. + await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1) + self.assertIsNone(should_start) + + # Wait for the remainder of stt_timeout. + await asyncio.sleep(stt_timeout - AGGREGATION_TIMEOUT + 0.1) + self.assertTrue(should_start) + + async def test_fallback_uses_only_user_speech_timeout(self): + """Fallback path (no VAD) ignores stt_timeout and uses only user_speech_timeout. + + stt_timeout is defined as "p99 after VAD stop" — without a VAD + reference point it has no meaning. The fallback measures + inactivity since the last transcript, which is user_speech_timeout. + """ + stt_timeout = AGGREGATION_TIMEOUT * 4 + strategy = SpeechTimeoutUserTurnStopStrategy(user_speech_timeout=AGGREGATION_TIMEOUT) + await strategy.setup(self.task_manager) + await strategy.process_frame( + STTMetadataFrame(service_name="test", ttfs_p99_latency=stt_timeout) + ) + + should_start = None + + @strategy.event_handler("on_user_turn_stopped") + async def on_user_turn_stopped(strategy, params): + nonlocal should_start + should_start = True + + # Transcript arrives without any VAD frame — fallback path. + await strategy.process_frame(TranscriptionFrame(text="Hello!", user_id="cat", timestamp="")) + + # The fallback timer is user_speech_timeout, not stt_timeout. + await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1) + self.assertTrue(should_start) + async def test_reset_clears_stale_text_no_premature_stop(self): """Test that reset() clears stale text and cancels timeout, preventing premature stop. diff --git a/tests/test_xai_tts.py b/tests/test_xai_tts.py index aab984567..98ac48fd4 100644 --- a/tests/test_xai_tts.py +++ b/tests/test_xai_tts.py @@ -4,14 +4,19 @@ # SPDX-License-Identifier: BSD 2-Clause License # -"""Tests for XAIHttpTTSService.""" +"""Tests for XAIHttpTTSService and XAITTSService.""" import asyncio +import base64 +import json import unittest +from urllib.parse import parse_qs, urlparse import aiohttp import pytest +import websockets from aiohttp import web +from websockets.asyncio.server import serve from pipecat.frames.frames import ( AggregatedTextFrame, @@ -21,7 +26,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, TTSTextFrame, ) -from pipecat.services.xai.tts import XAIHttpTTSService +from pipecat.services.xai.tts import XAIHttpTTSService, XAITTSService from pipecat.tests.utils import run_test @@ -87,5 +92,87 @@ async def test_run_xai_tts_success(aiohttp_client): } +@pytest.mark.asyncio +async def test_run_xai_websocket_tts_success(): + """xAI WS TTS should send text.delta+text.done and emit frames from audio.delta+audio.done.""" + + captured: dict = { + "request_path": None, + "auth_header": None, + "messages": [], + } + + audio_bytes = b"\x00\x01\x02\x03" * 1024 + + async def handler(ws): + request = ws.request + captured["request_path"] = request.path + captured["auth_header"] = request.headers.get("Authorization") + + try: + async for raw in ws: + msg = json.loads(raw) + captured["messages"].append(msg) + if msg.get("type") == "text.done": + await ws.send( + json.dumps( + { + "type": "audio.delta", + "delta": base64.b64encode(audio_bytes).decode("ascii"), + } + ) + ) + await ws.send(json.dumps({"type": "audio.done", "trace_id": "test-trace"})) + except websockets.ConnectionClosed: + pass + + async with serve(handler, "127.0.0.1", 0) as server: + host, port = next(iter(server.sockets)).getsockname()[:2] + base_url = f"ws://{host}:{port}/v1/tts" + + tts_service = XAITTSService( + api_key="test-key", + base_url=base_url, + sample_rate=24000, + ) + + down_frames, _ = await run_test( + tts_service, + frames_to_send=[TTSSpeakFrame(text="Hello from xAI."), _SleepAfterSpeak(0.3)], + ) + + frame_types = [type(frame) for frame in down_frames] + assert TTSStartedFrame in frame_types + assert TTSAudioRawFrame in frame_types + assert TTSStoppedFrame in frame_types + + audio_frames = [frame for frame in down_frames if isinstance(frame, TTSAudioRawFrame)] + assert audio_frames + assert all(frame.sample_rate == 24000 for frame in audio_frames) + assert all(frame.num_channels == 1 for frame in audio_frames) + assert b"".join(f.audio for f in audio_frames) == audio_bytes + + assert captured["auth_header"] == "Bearer test-key" + parsed = urlparse(captured["request_path"]) + query = parse_qs(parsed.query) + assert query["voice"] == ["eve"] + assert query["language"] == ["en"] + assert query["codec"] == ["pcm"] + assert query["sample_rate"] == ["24000"] + + types_sent = [m.get("type") for m in captured["messages"]] + assert "text.delta" in types_sent + assert "text.done" in types_sent + delta_msg = next(m for m in captured["messages"] if m.get("type") == "text.delta") + assert delta_msg["delta"] == "Hello from xAI." + + +# Small helper imported lazily to avoid circular import in fixture-lite tests. +def _SleepAfterSpeak(duration: float): + from pipecat.tests.utils import SleepFrame + + return SleepFrame(sleep=duration) + + if __name__ == "__main__": unittest.main() diff --git a/uv.lock b/uv.lock index 52a436cf3..113fb056c 100644 --- a/uv.lock +++ b/uv.lock @@ -4403,6 +4403,9 @@ websockets-base = [ whisper = [ { name = "faster-whisper" }, ] +xai = [ + { name = "websockets" }, +] [package.dev-dependencies] dev = [ @@ -4507,7 +4510,8 @@ requires-dist = [ { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'soniox'" }, { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'ultravox'" }, { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'websocket'" }, - { name = "pipecat-ai-small-webrtc-prebuilt", marker = "extra == 'runner'", specifier = ">=2.5.0" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'xai'" }, + { name = "pipecat-ai-small-webrtc-prebuilt", marker = "extra == 'runner'", specifier = ">=2.4.0" }, { name = "piper-tts", marker = "extra == 'piper'", specifier = ">=1.3.0,<2" }, { name = "protobuf", specifier = ">=6.31.1,<7" }, { name = "pvkoala", marker = "extra == 'koala'", specifier = "~=2.0.3" }, @@ -4522,7 +4526,7 @@ requires-dist = [ { name = "requests", marker = "extra == 'kokoro'", specifier = ">=2.32.5,<3" }, { name = "requests", marker = "extra == 'piper'", specifier = ">=2.32.5,<3" }, { name = "resampy", specifier = "~=0.4.3" }, - { name = "sarvamai", marker = "extra == 'sarvam'", specifier = "==0.1.26" }, + { name = "sarvamai", marker = "extra == 'sarvam'", specifier = "==0.1.28" }, { name = "sentry-sdk", marker = "extra == 'sentry'", specifier = ">=2.28.0,<3" }, { name = "simli-ai", marker = "extra == 'simli'", specifier = "~=2.0.1" }, { name = "soundfile", marker = "extra == 'soundfile'", specifier = "~=0.13.1" }, @@ -4569,14 +4573,14 @@ docs = [ [[package]] name = "pipecat-ai-small-webrtc-prebuilt" -version = "2.5.0" +version = "2.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "fastapi", extra = ["all"] }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2c/4f/40bfc9fc1a13f9b1f2657e292c51ff3e3516c530ca722effdcf342d465d9/pipecat_ai_small_webrtc_prebuilt-2.5.0.tar.gz", hash = "sha256:51481506b7b5dff10eff0357ff929cba504a5198c3393697178d2be9895ad9e6", size = 474299, upload-time = "2026-04-22T18:05:16.494Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/02/1e6e90f084ebb1fc954f37661c4614219e4c9fec3d305c8abe5141707b0c/pipecat_ai_small_webrtc_prebuilt-2.4.0.tar.gz", hash = "sha256:c5eddca4e061afb7c5f98cf52ccb85511978a8c834447f6c6d662029e02950c4", size = 472449, upload-time = "2026-03-13T14:17:08.164Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/34/58/1a2e10c1fb7b44e47558cb6c0954e24a60f98afe912fe55c74fdee66f080/pipecat_ai_small_webrtc_prebuilt-2.5.0-py3-none-any.whl", hash = "sha256:23b1eee95662a0072d9ee5128b8567108eda10d5a54ad71f279730afbb678bfe", size = 474308, upload-time = "2026-04-22T18:05:14.552Z" }, + { url = "https://files.pythonhosted.org/packages/25/77/8f6f67142a153943fff31530d51dcf7a2374c39dfa9aba6ef163bf0c622f/pipecat_ai_small_webrtc_prebuilt-2.4.0-py3-none-any.whl", hash = "sha256:9e9a3aa24231b1bf4101a6a2b42c4164a186c0c3d3e49bd51f77280eaa402d12", size = 472792, upload-time = "2026-03-13T14:17:06.556Z" }, ] [[package]] @@ -5891,7 +5895,7 @@ wheels = [ [[package]] name = "sarvamai" -version = "0.1.26" +version = "0.1.28" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, @@ -5900,9 +5904,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d7/31/13f65e8533b667514e1cfe838d12a14494cbc5943fd8f0c101305127459b/sarvamai-0.1.26.tar.gz", hash = "sha256:d51a213c27feb33d65f5b71e4882dcdb873dc5e0d720390b7ba18d1bdeec2471", size = 113050, upload-time = "2026-03-06T16:40:36.647Z" } +sdist = { url = "https://files.pythonhosted.org/packages/23/44/57a7a37be64953bb0bec9f674e92a9f8fb7070ce6aeb44c6f22720458b40/sarvamai-0.1.28.tar.gz", hash = "sha256:bc52f0c849e429d1a4493e49b1b4b65bf06965f3936d4eb6ee3da3130452e7ea", size = 139022, upload-time = "2026-04-20T08:05:12.185Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/76/c9/c03a807ace9cafbfe26418be995e4959142a55313c9f26564586e111f31d/sarvamai-0.1.26-py3-none-any.whl", hash = "sha256:39e79ba0932f4501a2aa28f84fd2de64d34fc9a7af2b0d4ead1efa617517b3bd", size = 229057, upload-time = "2026-03-06T16:40:35.584Z" }, + { url = "https://files.pythonhosted.org/packages/84/90/95cd195a3a2ae9a9973c6a05705207e8dd97b5aea90339bc24343cb54850/sarvamai-0.1.28-py3-none-any.whl", hash = "sha256:52e71c0a0f521552d4d948ec452699b44554a56e64ee354b14da2efc9d9046b3", size = 269335, upload-time = "2026-04-20T08:05:10.712Z" }, ] [[package]]