diff --git a/.github/workflows/generate-changelog.yml b/.github/workflows/generate-changelog.yml new file mode 100644 index 000000000..86bc7ae67 --- /dev/null +++ b/.github/workflows/generate-changelog.yml @@ -0,0 +1,174 @@ +name: Generate Changelog for Release + +on: + workflow_dispatch: + inputs: + version: + description: "Release version (e.g., 0.0.97)" + required: true + type: string + date: + description: "Release date (YYYY-MM-DD format, defaults to today)" + required: false + type: string + default: "" + +permissions: + contents: write + pull-requests: write + +jobs: + generate-changelog: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + + - name: Install dependencies + run: | + uv sync --group dev + + - name: Set release date + id: set_date + run: | + if [ -z "${{ inputs.date }}" ]; then + RELEASE_DATE=$(date +%Y-%m-%d) + echo "Using today's date: $RELEASE_DATE" + else + RELEASE_DATE="${{ inputs.date }}" + echo "Using provided date: $RELEASE_DATE" + fi + echo "release_date=$RELEASE_DATE" >> $GITHUB_OUTPUT + + - name: Validate inputs + run: | + # Validate version format (basic check) + if ! [[ "${{ inputs.version }}" =~ ^[0-9]+\.[0-9]+\.[0-9]+.*$ ]]; then + echo "Error: Version must be in format X.Y.Z (e.g., 0.0.97)" + exit 1 + fi + + # Validate date format if provided + if [ -n "${{ inputs.date }}" ]; then + if ! date -d "${{ inputs.date }}" >/dev/null 2>&1; then + # Try macOS date format + if ! date -j -f "%Y-%m-%d" "${{ inputs.date }}" >/dev/null 2>&1; then + echo "Error: Date must be in YYYY-MM-DD format (e.g., 2025-12-04)" + exit 1 + fi + fi + fi + + - name: Check for changelog fragments + id: check_fragments + run: | + FRAGMENT_COUNT=$(find changelog -name "*.md" ! -name "_template.md.j2" | wc -l | tr -d ' ') + echo "fragment_count=$FRAGMENT_COUNT" >> $GITHUB_OUTPUT + + if [ "$FRAGMENT_COUNT" -eq "0" ]; then + echo "❌ Error: No changelog fragments found in changelog/" + echo "" + echo "Cannot create a release without changelog entries." + echo "Add changelog fragments to the changelog/ directory (e.g., 1234.added.md) and try again." + exit 1 + fi + + # Validate fragment types + VALID_TYPES="added changed deprecated removed fixed security" + INVALID_FRAGMENTS="" + + for file in changelog/*.md; do + # Skip template + if [[ "$file" == "changelog/_template.md.j2" ]]; then + continue + fi + + # Extract type from filename (e.g., 1234.added.md -> added) + filename=$(basename "$file") + # Handle both 1234.added.md and 1234.added.2.md patterns + type=$(echo "$filename" | sed -E 's/^[0-9]+\.([a-z]+)(\.[0-9]+)?\.md$/\1/') + + # Check if type is valid + if ! echo "$VALID_TYPES" | grep -wq "$type"; then + INVALID_FRAGMENTS="$INVALID_FRAGMENTS\n - $filename (type: '$type')" + fi + done + + if [ -n "$INVALID_FRAGMENTS" ]; then + echo "❌ Error: Invalid changelog fragment types found:" + echo -e "$INVALID_FRAGMENTS" + echo "" + echo "Valid types are: $VALID_TYPES" + echo "Example: 1234.added.md, 5678.fixed.md" + exit 1 + fi + + echo "✓ Found $FRAGMENT_COUNT changelog fragment(s)" + echo "has_fragments=true" >> $GITHUB_OUTPUT + + - name: Preview changelog + run: | + echo "## Preview of changelog for version ${{ inputs.version }}" + echo "" + uv run towncrier build --draft --version "${{ inputs.version }}" --date "${{ steps.set_date.outputs.release_date }}" + + - name: Build changelog + run: | + uv run towncrier build --version "${{ inputs.version }}" --date "${{ steps.set_date.outputs.release_date }}" --yes + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v7 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: "Update changelog for version ${{ inputs.version }}" + title: "Release ${{ inputs.version }} - Changelog Update" + body: | + ## Changelog Update for Release ${{ inputs.version }} + + This PR updates the CHANGELOG.md with all changes for version **${{ inputs.version }}**. + + ### Summary + - **Version:** ${{ inputs.version }} + - **Date:** ${{ steps.set_date.outputs.release_date }} + - **Fragments processed:** ${{ steps.check_fragments.outputs.fragment_count }} + + ### What this PR does + - ✅ Adds new release section to CHANGELOG.md + - ✅ Removes processed changelog fragments + - ✅ Ready to merge for release + + ### Next Steps + 1. Review the changelog entries below + 2. Make any necessary edits to CHANGELOG.md if needed + 3. Merge this PR + 4. Continue with your release process + + --- + +
+ 📋 Preview of changes + + The changelog has been updated with entries from the following fragments: + + ```bash + ${{ steps.check_fragments.outputs.fragment_count }} fragments processed + ``` + +
+ branch: changelog-${{ inputs.version }} + delete-branch: true + labels: | + changelog + release diff --git a/CHANGELOG.md b/CHANGELOG.md index 10aa1c662..11bafc5b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,408 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] + + +## [0.0.97] - 2025-12-05 + +### Added + +- Added new Gradium services, `GradiumSTTService` and `GradiumTTSService`, for + speech-to-text and text-to-speech functionality using Gradium's API. + +- Additions for `AsyncAITTSService` and `AsyncAIHttpTTSService`: + + - Added new `languages`: `pt`, `nl`, `ar`, `ru`, `ro`, `ja`, `he`, `hy`, + `tr`, `hi`, `zh`. + - Updated the default model to `asyncflow_multilingual_v1.0` for improved + accuracy and broader language coverage. + +- Added optional tool and tool output filters for MCP services. + +### Changed + +- Updated Deepgram logging to include Deepgram request IDs for improved + debugging. + +- Text Aggregation Improvements: + + - **Breaking Change**: `BaseTextAggregator.aggregate()` now returns + `AsyncIterator[Aggregation]` instead of `Optional[Aggregation]`. This + enables the aggregator to return multiple results based on the provided + text. + - Refactored text aggregators to use inheritance: `SkipTagsAggregator` and + `PatternPairAggregator` now inherit from `SimpleTextAggregator`, reusing + the base class's sentence detection logic. + +- Improved interruption handling to prevent bots from repeating themselves. LLM + services that return multiple sentences in a single response (e.g., + `GoogleLLMService`) are now split into individual sentences before being sent + to TTS. This ensures interruptions occur at sentence boundaries, preventing + the bot from repeating content after being interrupted during long responses. + +- Updated `AICFilter` to use Quail STT as the default model + (`AICModelType.QUAIL_STT`). Quail STT is optimized for human-to-machine + interaction (e.g., voice agents, speech-to-text) and operates at a native + sample rate of 16 kHz with fixed enhancement parameters. + +- If an unexpected exception is caught, or if `FrameProcessor.push_error()` is + called with an exception, the file name and line number where the exception + occured are now logged. + +- Updated Smart Turn model weights to v3.1. + +- Smart Turn analyzer now uses the full context of the turn rather than just + the audio since VAD last triggered. + +- Updated `CartesiaSTTService` to return the full transcription `result` in the + `TranscriptionFrame` and `InterimTranscriptionFrame`. This provides access to + word timestamp data. + +- `HumeTTSService` changes: + + - Added tracking headers (`X-Hume-Client-Name` and `X-Hume-Client-Version`) + to all requests made by `HumeTTSService` to the Hume API for better usage + tracking and analytics. + - Added `stop()` and `cancel()` cleanup methods to `HumeTTSService` to + properly close the HTTP client and prevent resource leaks. + +### Deprecated + +- NVIDIA Services name changes (all functionality is unchanged): + + - `NimLLMService` is now deprecated, use `NvidiaLLMService` instead. + - `RivaSTTService` is now deprecated, use `NvidiaSTTService` instead. + - `RivaTTSService` is now deprecated, use `NvidiaTTSService` instead. + - Use `uv pip install pipecat-ai[nvidia]` instead of + `uv pip install pipecat-ai[riva]` + +- The `noise_gate_enable` parameter in `AICFilter` is deprecated and no longer + has any effect. Noise gating is now handled automatically by the AIC VAD + system. Use `AICFilter.create_vad_analyzer()` for VAD functionality instead. + +- Package `pipecat.sync` is deprecated, use `pipecat.utils.sync` instead. + +### Fixed + +- Fixed bug in `PatternPairAggregator` where pattern handlers could be called + multiple times for `KEEP` or `AGGREGATE` patterns. + +- Fixed sentence aggregation to correctly handle ambiguous punctuation in + streaming text, such as currency ("$29.95") and abbreviations ("Mr. Smith"). + +- Fixed an issue in `AWSTranscribeSTTService` where the `region` arg was always + set to `us-east-1` when providing an AWS_REGION env var. + +- Fixed an issue in `SarvamTTSService` where the last sentence was not being + spoken. Now, audio is flushed when the TTS services receives the + `LLMFullResponseEndFrame` or `EndFrame`. + +- Fixed an issue in `DeepgramTTSService` where a `TTSStoppedFrame` was + incorrectly pushed after a functional call. This caused an issue with the + voice-ui-kit's conversational panel rending of the LLM output after a + function call. + +- Fixed an issue where `LLMTextFrame.skip_tts` was being overwritten by LLM + services. + +- Fixed an issue that caused `WebsocketService` instances to attempt + reconnection during shutdown. + +- Fixed an issue in `ElevenLabsTTSService` where character usage metrics were + only reported on the first TTS generation per turn. + +## [0.0.96] - 2025-11-26 🦃 "Happy Thanksgiving!" 🦃 + +### Added + +- Added `AWSBedrockAgentCoreProcessor` to support invoking an AgentCore-hosted + agent in a Pipecat pipeline. + +- Enhanced error handling across the framework: + + - Added `on_error` callback to `FrameProcessor` for centralized error + handling. + + - Renamed `push_error(error: ErrorFrame)` to `push_error_frame(error: ErrorFrame)` + for clarity. + + - Added new `push_error` method for simplified error reporting: + + ```python + async def push_error(error_msg: str, + exception: Optional[Exception] = None, + fatal: bool = False) + ``` + + - Standardized error logging by replacing `logger.exception` calls with + `logger.error` throughout the codebase. + +- Added `cache_read_input_tokens`, `cache_creation_input_tokens` and + `reasoning_tokens` to OTel spans for LLM call + +- Added `LiveKitRESTHelper` utility class for managing LiveKit rooms via REST API. + +- Added `DeepgramSageMakerSTTService` which connects to a SageMaker hosted + Deepgram STT model. Added `07c-interruptible-deepgram-sagemaker.py` + foundational example. + +- Added `SageMakerBidiClient` to connect to SageMaker hosted BiDi compatible + services. + +- Added support for `include_timestamps` and `enable_logging` in + `ElevenLabsRealtimeSTTService`. When `include_timestamps` is enabled, + timestamp data is included in the `TranscriptionFrame`'s `result` + parameter. + +- Added optional speaking rate control to `InworldTTSService`. + +- Introduced a new `AggregatedTextFrame` type to support passing text along with + an `aggregated_by` field to describe the type of text + included. `TTSTextFrame`s now inherit from `AggregatedTextFrame`. With this + inheritance, an observer can watch for `AggregatedTextFrame`s to accumlate the + perceived output and determine whether or not the text was spoken based on if + that frame is also a `TTSTextFrame`. + + With this frame, the llm token stream can be transformed into custom + composable chunks, allowing for aggregation outside the TTS service. This + makes it possible to listen for or handle those aggregations and sets the + stage for doing things like composing a best effort of the perceived llm + output in a more digestable form and to do so whether or not it is processed + by a TTS or if even a TTS exists. + +- Introduced `LLMTextProcessor`: A new processor meant to allow customization + for how LLMTextFrames should be aggregated and considered. It's purpose is to + turn `LLMTextFrame`s into `AggregatedTextFrame`s. By default, a TTSService + will still aggregate `LLMTextFrame`s by sentence for the service to + consume. However, if you wish to override how the llm text is aggregated, you + should no longer override the TTS's internal text_aggregator, but instead, + insert this processor between your LLM and TTS in the pipeline. + +- New `bot-output` RTVI message to represent what the bot actually "says". + + - The `RTVIObserver` now emits `bot-output` messages based off the new + `AggregatedTextFrame`s (`bot-tts-text` and `bot-llm-text` are still + supported and generated, but `bot-transcript` is now deprecated in lieu of + this new, more thorough, message). + + - The new `RTVIBotOutputMessage` includes the fields: + + - `spoken`: A boolean indicating whether the text was spoken by TTS + + - `aggregated_by`: A string representing how the text was aggregated + ("sentence", "word", "my custom aggregation") + + - Introduced new fields to `RTVIObserver` to support the new `bot-output` + messaging: + + - `bot_output_enabled`: Defaults to True. Set to false to disable bot-output + messages. + + - `skip_aggregator_types`: Defaults to `None`. Set to a list of strings that + match aggregation types that should not be included in bot-output + messages. (Ex. `credit_card`) + + - Introduced new methods, `add_text_transformer()` and + `remove_text_transformer()`, to `RTVIObserver` to support providing (and + subsequently removing) callbacks for various types of aggregations (or all + aggregations with `*`) that can modify the text before being sent as a + `bot-output` or `tts-text` message. (Think obscuring the credit card or + inserting extra detail the client might want that the context doesn't need.) + +- In `MiniMaxHttpTTSService`: + + - Added support for speech-2.6-hd and speech-2.6-turbo models + + - Added languages: Afrikaans, Bulgarian, Catalan, Danish, Persian, Filipino, + Hebrew, Croatian, Hungarian, Malay, Norwegian, Nynorsk, Slovak, Slovenian, + Swedish, and Tamil + + - Added new emotions: calm and fluent + +- Added `enable_logging` to `SimliVideoService` input parameters. It's disabled + by default. + +### Changed + +- Updated `FishAudioTTSService` default model to `s1`. + +- Updated `DeepgramTTSService` to use Deepgram's TTS websocket API. ⚠️ This is + a potential breaking change, which only affects you if you're self-hosting + `DeepgramTTSService`. The new service uses Websockets and improves TTFB + latency. + +- Updated `daily-python` to 0.22.0. + +- `BaseTextAggregator` changes: + + Modified the BaseTextAggregator type so that when text gets aggregated, + metadata can be associated with it. Currently, that just means a `type`, so + that the aggregation can be classified or described. Changes made to support + this: + + - ⚠️ IMPORTANT: Aggregators are now expected to strip leading/trailing white + space characters before returning their aggregation from `aggregation()` or + `.text`. This way all aggregators have a consistent contract allowing + downstream use to know how to stitch aggregations back together. + + - Introduced a new `Aggregation` dataclass to represent both the aggregated + `text` and a string identifying the `type` of aggregation (ex. "sentence", + "word", "my custom aggregation") + + - ⚠️ Breaking change: `BaseTextAggregator.text` now returns an `Aggregation` + (instead of `str`). + + Before: + + ```python + aggregated_text = myAggregator.text + ``` + + Now: + + ```python + aggregated_text = myAggregator.text.text + ``` + + - ⚠️ Breaking change: `BaseTextAggregator.aggregate()` now returns + `Optional[Aggregation]` (instead of `Optional[str]`). + + Before: + + ```python + aggregation = myAggregator.aggregate(text) + print(f"successfully aggregated text: {aggregation}") + ``` + + Now: + + ```python + aggregation = myAggregator.aggregate(text) + if aggregation: + print(f"successfully aggregated text: {aggregation.text}") + ``` + + - `SimpleTextAggregator`, `SkipTagsAggregator`, `PatternPairAggregator` + updated to produce/consume `Aggregation` objects. + + - All uses of the above Aggregators have been updated accordingly. + +- Augmented the `PatternPairAggregator` so that matched patterns can be treated + as their own aggregation, taking advantage of the new. To that end: + + - Introduced a new, preferred version of `add_pattern` to support a new option + for treating a match as a separate aggregation returned from + `aggregate()`. This replaces the now deprecated `add_pattern_pair` method + and you provide a `MatchAction` in lieu of the `remove_match` field. + + - `MatchAction` enum: `REMOVE`, `KEEP`, `AGGREGATE`, allowing customization + for how a match should be handled. + + - `REMOVE`: The text along with its delimiters will be removed from the + streaming text. Sentence aggregation will continue on as if this text + did not exist. + + - `KEEP`: The delimiters will be removed, but the content between them + will be kept. Sentence aggregation will continue on with the internal + text included. + + - `AGGREGATE`: The delimiters will be removed and the content between will + be treated as a separate aggregation. Any text before the start of the + pattern will be returned early, whether or not a complete sentence was + found. Then the pattern will be returned. Then the aggregation will + continue on sentence matching after the closing delimiter is found. The + content between the delimiters is not aggregated by sentence. It is + aggregated as one single block of text. + + - `PatternMatch` now extends `Aggregation` and provides richer info to + handlers. + + - ⚠️ Breaking change: The `PatternMatch` type returned to handlers registered + via `on_pattern_match` has been updated to subclass from the new + `Aggregation` type, which means that `content` has been replaced with + `text` and `pattern_id` has been replaced with `type`: + + ```python + async dev on_match_tag(match: PatternMatch): + pattern = match.type # instead of match.pattern_id + text = match.text # instead of match.content + ``` + +- `TextFrame` now includes the field `append_to_context` to support setting + whether or not the encompassing text should be added to the LLM context (by + the LLM assistant aggregator). It defaults to `True`. + +- `TTSService` base class updates: + + - `TTSService`s now accept a new `skip_aggregator_types` to avoid speaking + certain aggregation types (now determined/returned by the aggregator) + + - Introduced the ability to do a just-in-time transform of text before it gets + sent to the TTS service via callbacks you can set up via a new init field, + `text_transforms` or a new method `add_text_transformer()`. This makes it + possible to do things like introduce TTS-specific tags for spelling or + emotion or change the pronunciation of something on the + fly. `remove_text_transformer` has also been added to support removing a + registered transform callback. + + - TTS services push `AggregatedTextFrame` in addition to `TTSTextFrame`s when + either an aggregation occurs that should not be spoken or when the TTS + service supports word-by-word timestamping. In the latter case, the + `TTSService` preliminarily generates an `AggregatedTextFrame`, aggregated by + sentence to generate the full sentence content as early as possible. + +- Updated `CartesiaTTSService`: + + - Modified use of custom default text_aggregator to avoid deprecation warnings + and push users towards use of transformers or the `LLMTextProcessor` + + - Added convenience methods for taking advantage of Cartesia's SSML tags: + spell, emotion, pauses, volume, and speed. + +- Updated `RimeTTSService`: + + - Modified use of custom default text_aggregator to avoid deprecation warnings + and push users towards use of transformers or the `LLMTextProcessor` + + - Added convenience methods for taking advantage of Rime's customization + options: spell, pauses, pronunciations, and inline speed control. + +### Deprecated + +- The TTS constructor field, `text_aggregator` is deprecated in favor of the new + `LLMTextProcessor`. TTSServices still have an internal aggregator for support + of default behavior, but if you want to override the aggregation behavior, you + should use the new processor. + +- The RTVI `bot-transcription` event is deprecated in favor of the new + `bot-output` message which is the canonical representation of bot output + (spoken or not). The code still emits a transcription message for backwards + compatibility while transition occurs. + +- Deprecated `add_pattern_pair` in the `PatternPairAggregator` which takes a + `pattern_id` and `remove_match` field in favor of the new `add_pattern` method + which takes a `type` and an `action` + +- `english_normalization` input parameter for `MiniMaxHttpTTSService` is + deprecated, use `test_normalization` instead. + +### Fixed + +- Fixed an issue in `AWSBedrockLLMService` where the `aws_region` arg was + always set to `us-east-1` when providing an AWS_REGION env var. + +- Fixed an issue with `DeepgramFluxSTTService` where it sometimes failed to reconnect. + +- Fixed an issue in `ElevenLabsRealtimeSTTService` where dynamic language + updates were not working. + +- Fixed an issue in `ElevenLabsRealtimeSTTService` where setting the sample + rate would result in transcripts failing. + +- Fixed `InworldTTSService` audio config payload to use camelCase keys expected + by the Inworld API. + +## [0.0.95] - 2025-11-18 ### Added @@ -22,15 +423,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `ElevenLabsRealtimeSTTService` which implements the Realtime STT service from ElevenLabs. -- Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and - example wiring; leverages the enhancement model for robust detection with no - ONNX dependency or added processing complexity. +- Added word-level timestamps support to Hume TTS service ### Changed -- ⚠️ Breaking change: `LLMContext.create_image_message()` and - `LLMContext.create_audio_message()` are now async methods. This fixes and - issue where the asyncio event loop would be blocked while encoding audio or +- ⚠️ Breaking change: `LLMContext.create_image_message()`, + `LLMContext.create_audio_message()`, `LLMContext.add_image_frame_message()` + and `LLMContext.add_audio_frames_message()` are now async methods. This fixes + an issue where the asyncio event loop would be blocked while encoding audio or images. - `ConsumerProcessor` now queues frames from the producer internally instead of @@ -71,10 +471,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- Fixed a race condition where, if the LLM received instructions to both produce - text and invoke a function call at the same time, the context would not be - updated before the function call result arrived, causing the bot to repeat - itself. +- Fixed a `SimliVideoService` connection issue. - Fixed an issue in the `Runner` where, when using `SmallWebRTCTransport`, the `request_data` was not being passed to the `SmallWebRTCRunnerArguments` body. diff --git a/COMMUNITY_INTEGRATIONS.md b/COMMUNITY_INTEGRATIONS.md index 080d75ef2..a26836a52 100644 --- a/COMMUNITY_INTEGRATIONS.md +++ b/COMMUNITY_INTEGRATIONS.md @@ -79,7 +79,7 @@ Once your PR is submitted, post in the `#community-integrations` Discord channel **Examples:** -- [RivaSTTService](https://github.com/pipecat-ai/pipecat/blob/main/src/pipecat/services/riva/stt.py) +- [NvidiaSTTService](https://github.com/pipecat-ai/pipecat/blob/main/src/pipecat/services/nvidia/stt.py) - [FalSTTService](https://github.com/pipecat-ai/pipecat/blob/main/src/pipecat/services/fal/stt.py) #### Key requirements: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7a17dff7a..490966ddf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,24 +17,121 @@ We welcome contributions of all kinds! Your help is appreciated. Follow these st git checkout -b your-branch-name ``` 4. **Make your changes**: Edit or add files as necessary. -5. **Test your changes**: Ensure that your changes look correct and follow the style set in the codebase. -6. **Commit your changes**: Once you're satisfied with your changes, commit them with a meaningful message. +5. **Add a changelog entry**: Create a changelog fragment file (see [Changelog Entries](#changelog-entries) below). +6. **Test your changes**: Ensure that your changes look correct and follow the style set in the codebase. +7. **Commit your changes**: Once you're satisfied with your changes, commit them with a meaningful message. ```bash git commit -m "Description of your changes" ``` -7. **Push your changes**: Push your branch to your forked repository. +8. **Push your changes**: Push your branch to your forked repository. ```bash git push origin your-branch-name ``` -8. **Submit a Pull Request (PR)**: Open a PR from your forked repository to the main branch of this repo. +9. **Submit a Pull Request (PR)**: Open a PR from your forked repository to the main branch of this repo. > Important: Describe the changes you've made clearly! Our maintainers will review your PR, and once everything is good, your contributions will be merged! +## Changelog Entries + +Every pull request that makes a user-facing change should include a changelog entry. We use a changelog fragment system to avoid merge conflicts. + +### Creating a Changelog Fragment + +1. Create a new file in the `changelog/` directory with this naming pattern: + + ``` + ..md + ``` + +2. Choose the appropriate type: + + - `added.md` - New features + - `changed.md` - Changes in existing functionality + - `deprecated.md` - Soon-to-be removed features + - `removed.md` - Removed features + - `fixed.md` - Bug fixes + - `security.md` - Security fixes + +3. Write your changelog entry as a Markdown bullet point. Include the `-` at the start: + +**Example files:** + +`changelog/1234.added.md`: + +```markdown +- Added support for Anthropic Claude 3.5 Sonnet with improved streaming performance. +``` + +`changelog/5678.fixed.md`: + +```markdown +- Fixed an issue where audio frames were dropped during high-load scenarios. +``` + +**For entries with nested bullets:** + +`changelog/1234.changed.md`: + +```markdown +- Updated service configuration: + + - Changed default timeout to 30 seconds + - Added retry logic for failed connections +``` + +### Multiple Changes in One PR + +**Different types of changes:** Create separate fragment files for each type: + +``` +changelog/1234.added.md +changelog/1234.fixed.md +``` + +**Multiple changes of the same type:** Create numbered fragment files: + +``` +changelog/1234.changed.md +changelog/1234.changed.2.md +``` + +**Related changes:** Use nested bullets in a single fragment: + +```markdown +- Updated service configuration: + + - Changed default timeout to 30 seconds + - Added retry logic for failed connections +``` + +**Rule of thumb:** One logical change per fragment file. If changes are unrelated, use separate files. + +### Preview Your Changes + +To see what your changelog entry will look like: + +```bash +towncrier build --draft --version Unreleased +``` + +This won't modify any files, just show you a preview. + +### When to Skip Changelog Entries + +You can skip adding a changelog entry for: + +- Documentation-only changes +- Internal refactoring with no user-facing impact +- Test-only changes +- CI/build configuration changes + +If you're unsure whether your change needs a changelog entry, ask in your PR! + ## Dependency Management This project uses [uv](https://docs.astral.sh/uv/) for dependency management. The `uv.lock` file is committed to ensure reproducible builds. diff --git a/README.md b/README.md index ffe826025..71e7d47c8 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,6 @@ [![PyPI](https://img.shields.io/pypi/v/pipecat-ai)](https://pypi.org/project/pipecat-ai) ![Tests](https://github.com/pipecat-ai/pipecat/actions/workflows/tests.yaml/badge.svg) [![codecov](https://codecov.io/gh/pipecat-ai/pipecat/graph/badge.svg?token=LNVUIVO4Y9)](https://codecov.io/gh/pipecat-ai/pipecat) [![Docs](https://img.shields.io/badge/Documentation-blue)](https://docs.pipecat.ai) [![Discord](https://img.shields.io/discord/1239284677165056021)](https://discord.gg/pipecat) [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/pipecat-ai/pipecat) -[![](https://getmanta.ai/api/badges?text=Manta%20Graph&link=manta)](https://getmanta.ai/pipecat) # 🎙️ Pipecat: Real-Time Voice & Multimodal AI Agents @@ -74,9 +73,9 @@ Catch new features, interviews, and how-tos on our [Pipecat TV](https://www.yout | Category | Services | | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Gradium](https://docs.pipecat.ai/server/services/stt/gradium), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | | LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova) [Together AI](https://docs.pipecat.ai/server/services/llm/together) | -| Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [Speechmatics](https://docs.pipecat.ai/server/services/tts/speechmatics), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | +| Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Gradium](https://docs.pipecat.ai/server/services/tts/gradium), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [Speechmatics](https://docs.pipecat.ai/server/services/tts/speechmatics), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | | Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | | Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [SmallWebRTCTransport](https://docs.pipecat.ai/server/services/transport/small-webrtc), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local | | Serializers | [Plivo](https://docs.pipecat.ai/server/utilities/serializers/plivo), [Twilio](https://docs.pipecat.ai/server/utilities/serializers/twilio), [Telnyx](https://docs.pipecat.ai/server/utilities/serializers/telnyx) | diff --git a/changelog/3189.added.md b/changelog/3189.added.md new file mode 100644 index 000000000..5dbcb6058 --- /dev/null +++ b/changelog/3189.added.md @@ -0,0 +1,6 @@ +- Data and control frames can now be marked as non-interruptible by using the + `UninterruptibleFrame` mixin. Frames marked as `UninterruptibleFrame` will not + be interrupted during processing, and any queued frames of this type will be + retained in the internal queues. This is useful when you need ordered frames + (data or control) that should not be discarded or cancelled due to + interruptions. diff --git a/changelog/3189.changed.md b/changelog/3189.changed.md new file mode 100644 index 000000000..f8f24a856 --- /dev/null +++ b/changelog/3189.changed.md @@ -0,0 +1,3 @@ +- `FunctionCallInProgressFrame` and `FunctionCallResultFrame` have changed from + system frames to a control frame and a data frame, respectively, and are now + both marked as `UninterruptibleFrame`. diff --git a/changelog/_template.md.j2 b/changelog/_template.md.j2 new file mode 100644 index 000000000..56b2e60c8 --- /dev/null +++ b/changelog/_template.md.j2 @@ -0,0 +1,16 @@ +{% for section, _ in sections.items() %} +{% if sections[section] %} +{% for category, val in definitions.items() if category in sections[section]%} +### {{ definitions[category]['name'] }} + +{% for text, values in sections[section][category].items() %} +{{ text }} + +{% endfor %} +{% endfor %} +{% else %} +No significant changes. + +{% endif %} +{% endfor %} + diff --git a/docs/api/conf.py b/docs/api/conf.py index 8946a34d3..c92eaa97d 100644 --- a/docs/api/conf.py +++ b/docs/api/conf.py @@ -119,7 +119,6 @@ def import_core_modules(): "pipecat.observers", "pipecat.runner", "pipecat.serializers", - "pipecat.sync", "pipecat.transcriptions", "pipecat.utils", ] diff --git a/docs/api/index.rst b/docs/api/index.rst index df84d9580..85f373db7 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -30,7 +30,6 @@ Quick Links Runner Serializers Services - Sync Transcriptions Transports - Utils \ No newline at end of file + Utils diff --git a/env.example b/env.example index 2865772ea..98ddec262 100644 --- a/env.example +++ b/env.example @@ -44,6 +44,7 @@ DAILY_SAMPLE_ROOM_URL=https://... # Deepgram DEEPGRAM_API_KEY=... +SAGEMAKER_ENDPOINT_NAME=... # DeepSeek DEEPSEEK_API_KEY=... @@ -72,6 +73,9 @@ GOOGLE_CLOUD_PROJECT_ID=... GOOGLE_CLOUD_LOCATION=... GOOGLE_TEST_CREDENTIALS=... +# Gradium +GRAPDIUM_API_KEY=... + # Grok GROK_API_KEY=... @@ -190,4 +194,4 @@ TWILIO_AUTH_TOKEN=... WHATSAPP_TOKEN=... WHATSAPP_WEBHOOK_VERIFICATION_TOKEN=... WHATSAPP_PHONE_NUMBER_ID=... -WHATSAPP_APP_SECRET=... \ No newline at end of file +WHATSAPP_APP_SECRET=... diff --git a/examples/foundational/01c-fastpitch.py b/examples/foundational/01c-nvidia-riva-tts.py similarity index 94% rename from examples/foundational/01c-fastpitch.py rename to examples/foundational/01c-nvidia-riva-tts.py index 3a239a3fd..7063472d0 100644 --- a/examples/foundational/01c-fastpitch.py +++ b/examples/foundational/01c-nvidia-riva-tts.py @@ -15,7 +15,7 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.riva.tts import FastPitchTTSService +from pipecat.services.nvidia.tts import NvidiaTTSService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams @@ -36,7 +36,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - tts = FastPitchTTSService(api_key=os.getenv("NVIDIA_API_KEY")) + tts = NvidiaTTSService(api_key=os.getenv("NVIDIA_API_KEY")) task = PipelineTask( Pipeline([tts, transport.output()]), diff --git a/examples/foundational/07ae-interruptible-hume.py b/examples/foundational/07ae-interruptible-hume.py index 046f2d4c8..c5de34c85 100644 --- a/examples/foundational/07ae-interruptible-hume.py +++ b/examples/foundational/07ae-interruptible-hume.py @@ -13,24 +13,29 @@ from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame +from pipecat.frames.frames import LLMRunFrame, TTSTextFrame +from pipecat.observers.loggers.debug_log_observer import DebugLogObserver, FrameEndpoint from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, +) from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.hume.tts import HUME_SAMPLE_RATE, HumeTTSService from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams load_dotenv(override=True) + # We store functions so objects (e.g. SileroVADAnalyzer) don't get # instantiated. The function will be called when the desired transport gets # selected. @@ -88,7 +93,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): stt, context_aggregator.user(), # User responses llm, # LLM - tts, # TTS + tts, # TTS (HumeTTSService with word timestamps) transport.output(), # Transport bot output context_aggregator.assistant(), # Assistant spoken responses ] @@ -102,7 +107,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): audio_out_sample_rate=HUME_SAMPLE_RATE, ), idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - observers=[RTVIObserver(rtvi)], + observers=[ + RTVIObserver(rtvi), + DebugLogObserver( + frame_types={ + TTSTextFrame: (BaseOutputTransport, FrameEndpoint.SOURCE), + } + ), + ], ) @rtvi.event_handler("on_client_ready") @@ -112,6 +124,9 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info(f"Client connected") + logger.info( + "💡 Word timestamps are enabled! Watch the console for TTSTextFrame logs showing each word with its PTS." + ) # Kick off the conversation. messages.append({"role": "system", "content": "Please introduce yourself to the user."}) await task.queue_frames([LLMRunFrame()]) diff --git a/examples/foundational/39a-mcp-run-sse.py b/examples/foundational/07af-interruptible-gradium.py similarity index 61% rename from examples/foundational/39a-mcp-run-sse.py rename to examples/foundational/07af-interruptible-gradium.py index 328af6ce4..9ad3bcebb 100644 --- a/examples/foundational/39a-mcp-run-sse.py +++ b/examples/foundational/07af-interruptible-gradium.py @@ -4,12 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # - import os from dotenv import load_dotenv from loguru import logger -from mcp.client.session_group import SseServerParameters from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 @@ -23,10 +21,9 @@ from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.anthropic.llm import AnthropicLLMService -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.services.mcp_service import MCPClient +from pipecat.services.gradium.stt import GradiumSTTService +from pipecat.services.gradium.tts import GradiumTTSService +from pipecat.services.openai.llm import OpenAILLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams @@ -61,56 +58,34 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = GradiumSTTService(api_key=os.getenv("GRADIUM_API_KEY")) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + tts = GradiumTTSService( + api_key=os.getenv("GRADIUM_API_KEY"), + voice_id="YTpq7expH9539ERJ", ) - llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-7-sonnet-latest" - ) + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) - try: - # https://docs.mcp.run/integrating/tutorials/mcp-run-sse-openai-agents/ - mcp = MCPClient(server_params=SseServerParameters(url=os.getenv("MCP_RUN_SSE_URL"))) - except Exception as e: - logger.error(f"error setting up mcp") - logger.exception("error trace:") + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] - tools = {} - try: - tools = await mcp.register_tools(llm) - except Exception as e: - logger.error(f"error registering tools") - logger.exception("error trace:") - - system = f""" - You are a helpful LLM in a WebRTC call. - Your goal is to demonstrate your capabilities in a succinct way. - You have access to a number of tools provided by mcp.run. Use any and all tools to help users. - Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. - Respond to what the user said in a creative and helpful way. - When asked for today's date, use 'https://www.datetoday.net/'. - Don't overexplain what you are doing. - Just respond with short sentences when you are carrying out tool calls. - """ - - messages = [{"role": "system", "content": system}] - - context = LLMContext(messages, tools) + context = LLMContext(messages) context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ transport.input(), # Transport user input stt, - context_aggregator.user(), # User spoken responses + context_aggregator.user(), # User responses llm, # LLM tts, # TTS transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses and tool context + context_aggregator.assistant(), # Assistant spoken responses ] ) @@ -125,8 +100,9 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): - logger.info(f"Client connected: {client}") + logger.info(f"Client connected") # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") @@ -146,14 +122,6 @@ async def bot(runner_args: RunnerArguments): if __name__ == "__main__": - if not os.getenv("MCP_RUN_SSE_URL"): - logger.error( - f"Please set MCP_RUN_SSE_URL environment variable for this example. See https://mcp.run" - ) - import sys - - sys.exit(1) - from pipecat.runner.run import main main() diff --git a/examples/foundational/07c-interruptible-deepgram-sagemaker.py b/examples/foundational/07c-interruptible-deepgram-sagemaker.py new file mode 100644 index 000000000..db230a8ba --- /dev/null +++ b/examples/foundational/07c-interruptible-deepgram-sagemaker.py @@ -0,0 +1,137 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams +from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.aws.llm import AWSBedrockLLMService +from pipecat.services.deepgram.stt_sagemaker import DeepgramSageMakerSTTService +from pipecat.services.deepgram.tts import DeepgramTTSService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + # Initialize Deepgram SageMaker STT Service + # This requires: + # - AWS credentials configured (via environment variables or AWS CLI) + # - A deployed SageMaker endpoint with Deepgram model + stt = DeepgramSageMakerSTTService( + endpoint_name=os.getenv("SAGEMAKER_ENDPOINT_NAME"), + region=os.getenv("AWS_REGION"), + ) + + tts = DeepgramTTSService(api_key=os.getenv("DEEPGRAM_API_KEY"), voice="aura-2-andromeda-en") + + llm = AWSBedrockLLMService( + aws_region=os.getenv("AWS_REGION"), + model="us.amazon.nova-pro-v1:0", + params=AWSBedrockLLMService.InputParams(temperature=0.8), + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/07r-interruptible-riva-nim.py b/examples/foundational/07r-interruptible-nvidia.py similarity index 90% rename from examples/foundational/07r-interruptible-riva-nim.py rename to examples/foundational/07r-interruptible-nvidia.py index a9c1f74fd..bba99ea4c 100644 --- a/examples/foundational/07r-interruptible-riva-nim.py +++ b/examples/foundational/07r-interruptible-nvidia.py @@ -22,9 +22,9 @@ from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.nim.llm import NimLLMService -from pipecat.services.riva.stt import RivaSTTService -from pipecat.services.riva.tts import RivaTTSService +from pipecat.services.nvidia.llm import NvidiaLLMService +from pipecat.services.nvidia.stt import NvidiaSTTService +from pipecat.services.nvidia.tts import NvidiaTTSService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams @@ -59,11 +59,13 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = RivaSTTService(api_key=os.getenv("NVIDIA_API_KEY")) + stt = NvidiaSTTService(api_key=os.getenv("NVIDIA_API_KEY")) - llm = NimLLMService(api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.1-405b-instruct") + llm = NvidiaLLMService( + api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.1-405b-instruct" + ) - tts = RivaTTSService(api_key=os.getenv("NVIDIA_API_KEY")) + tts = NvidiaTTSService(api_key=os.getenv("NVIDIA_API_KEY")) messages = [ { diff --git a/examples/foundational/14i-function-calling-fireworks.py b/examples/foundational/14i-function-calling-fireworks.py index 235dcf8cc..87adfec41 100644 --- a/examples/foundational/14i-function-calling-fireworks.py +++ b/examples/foundational/14i-function-calling-fireworks.py @@ -76,7 +76,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = FireworksLLMService( api_key=os.getenv("FIREWORKS_API_KEY"), - model="accounts/fireworks/models/llama-v3p1-405b-instruct", + model="accounts/fireworks/models/gpt-oss-20b", ) # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. diff --git a/examples/foundational/14j-function-calling-nim.py b/examples/foundational/14j-function-calling-nvidia.py similarity index 97% rename from examples/foundational/14j-function-calling-nim.py rename to examples/foundational/14j-function-calling-nvidia.py index 97841f1a1..d18827726 100644 --- a/examples/foundational/14j-function-calling-nim.py +++ b/examples/foundational/14j-function-calling-nvidia.py @@ -27,7 +27,7 @@ from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.llm_service import FunctionCallParams -from pipecat.services.nim.llm import NimLLMService +from pipecat.services.nvidia.llm import NvidiaLLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams @@ -75,11 +75,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # text_filters=[MarkdownTextFilter()], ) - llm = NimLLMService( + llm = NvidiaLLMService( api_key=os.getenv("NVIDIA_API_KEY"), model="nvidia/llama-3.3-nemotron-super-49b-v1.5", # Recommended when turning thinking off - params=NimLLMService.InputParams(temperature=0.0), + params=NvidiaLLMService.InputParams(temperature=0.0), ) # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. diff --git a/examples/foundational/19-openai-realtime.py b/examples/foundational/19-openai-realtime.py index dca28b9fb..b7d99f525 100644 --- a/examples/foundational/19-openai-realtime.py +++ b/examples/foundational/19-openai-realtime.py @@ -14,20 +14,13 @@ from loguru import logger from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema -from pipecat.adapters.services.open_ai_realtime_adapter import OpenAIRealtimeLLMAdapter from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import ( - LLMRunFrame, - LLMSetToolsFrame, - LLMUpdateSettingsFrame, - TranscriptionMessage, -) +from pipecat.frames.frames import LLMRunFrame, LLMSetToolsFrame, TranscriptionMessage from pipecat.observers.loggers.transcription_log_observer import TranscriptionLogObserver from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response import LLMAssistantAggregatorParams from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments diff --git a/examples/foundational/19a-azure-realtime.py b/examples/foundational/19a-azure-realtime.py index b56c05af9..7b07985be 100644 --- a/examples/foundational/19a-azure-realtime.py +++ b/examples/foundational/19a-azure-realtime.py @@ -19,7 +19,6 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response import LLMAssistantAggregatorParams from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport diff --git a/examples/foundational/22-natural-conversation.py b/examples/foundational/22-natural-conversation.py index f95a473df..c4326ac4d 100644 --- a/examples/foundational/22-natural-conversation.py +++ b/examples/foundational/22-natural-conversation.py @@ -28,10 +28,10 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.llm_service import LLMService from pipecat.services.openai.llm import OpenAIContextAggregatorPair, OpenAILLMService -from pipecat.sync.event_notifier import EventNotifier from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams +from pipecat.utils.sync.event_notifier import EventNotifier load_dotenv(override=True) diff --git a/examples/foundational/22b-natural-conversation-proposal.py b/examples/foundational/22b-natural-conversation-proposal.py index 935676f9b..ca3737ebd 100644 --- a/examples/foundational/22b-natural-conversation-proposal.py +++ b/examples/foundational/22b-natural-conversation-proposal.py @@ -45,11 +45,11 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.llm_service import FunctionCallParams, LLMService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.sync.base_notifier import BaseNotifier -from pipecat.sync.event_notifier import EventNotifier from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams +from pipecat.utils.sync.base_notifier import BaseNotifier +from pipecat.utils.sync.event_notifier import EventNotifier from pipecat.utils.time import time_now_iso8601 load_dotenv(override=True) diff --git a/examples/foundational/22c-natural-conversation-mixed-llms.py b/examples/foundational/22c-natural-conversation-mixed-llms.py index cdcd21289..48fa7dfe0 100644 --- a/examples/foundational/22c-natural-conversation-mixed-llms.py +++ b/examples/foundational/22c-natural-conversation-mixed-llms.py @@ -46,11 +46,11 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.llm_service import FunctionCallParams, LLMService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.sync.base_notifier import BaseNotifier -from pipecat.sync.event_notifier import EventNotifier from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams +from pipecat.utils.sync.base_notifier import BaseNotifier +from pipecat.utils.sync.event_notifier import EventNotifier from pipecat.utils.time import time_now_iso8601 load_dotenv(override=True) diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 7a7155297..9654dd154 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -47,11 +47,11 @@ from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.google.llm import GoogleLLMService from pipecat.services.llm_service import LLMService -from pipecat.sync.base_notifier import BaseNotifier -from pipecat.sync.event_notifier import EventNotifier from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams +from pipecat.utils.sync.base_notifier import BaseNotifier +from pipecat.utils.sync.event_notifier import EventNotifier from pipecat.utils.time import time_now_iso8601 load_dotenv(override=True) @@ -391,7 +391,7 @@ class AudioAccumulator(FrameProcessor): ) self._user_speaking = False context = LLMContext() - context.add_audio_frames_message(audio_frames=self._audio_frames) + await context.add_audio_frames_message(audio_frames=self._audio_frames) await self.push_frame(LLMContextFrame(context=context)) elif isinstance(frame, InputAudioRawFrame): # Append the audio frame to our buffer. Treat the buffer as a ring buffer, dropping the oldest diff --git a/examples/foundational/35-pattern-pair-voice-switching.py b/examples/foundational/35-pattern-pair-voice-switching.py index 7ed9eb268..3a102acfd 100644 --- a/examples/foundational/35-pattern-pair-voice-switching.py +++ b/examples/foundational/35-pattern-pair-voice-switching.py @@ -62,7 +62,11 @@ from pipecat.services.openai.llm import OpenAILLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams -from pipecat.utils.text.pattern_pair_aggregator import PatternMatch, PatternPairAggregator +from pipecat.utils.text.pattern_pair_aggregator import ( + MatchAction, + PatternMatch, + PatternPairAggregator, +) load_dotenv(override=True) @@ -106,16 +110,16 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): pattern_aggregator = PatternPairAggregator() # Add pattern for voice switching - pattern_aggregator.add_pattern_pair( - pattern_id="voice_tag", + pattern_aggregator.add_pattern( + type="voice", start_pattern="", end_pattern="", - remove_match=True, + action=MatchAction.REMOVE, # Remove tags from final text ) # Register handler for voice switching async def on_voice_tag(match: PatternMatch): - voice_name = match.content.strip().lower() + voice_name = match.text.strip().lower() if voice_name in VOICE_IDS: # First flush any existing audio to finish the current context await tts.flush_audio() @@ -125,7 +129,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): else: logger.warning(f"Unknown voice: {voice_name}") - pattern_aggregator.on_pattern_match("voice_tag", on_voice_tag) + pattern_aggregator.on_pattern_match("voice", on_voice_tag) stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) diff --git a/examples/foundational/39-mcp-stdio.py b/examples/foundational/39-mcp-stdio.py index b88ee30b1..8fe64f73b 100644 --- a/examples/foundational/39-mcp-stdio.py +++ b/examples/foundational/39-mcp-stdio.py @@ -64,11 +64,14 @@ class UrlToImageProcessor(FrameProcessor): await self.push_frame(frame, direction) def extract_url(self, text: str): - data = json.loads(text) - if "artObject" in data: - return data["artObject"]["webImage"]["url"] - if "artworks" in data and len(data["artworks"]): - return data["artworks"][0]["webImage"]["url"] + try: + data = json.loads(text) + if "artObject" in data: + return data["artObject"]["webImage"]["url"] + if "artworks" in data and len(data["artworks"]): + return data["artworks"][0]["webImage"]["url"] + except: + pass return None @@ -88,6 +91,23 @@ class UrlToImageProcessor(FrameProcessor): logger.error(error_msg) +# full list of tools available from rijksmuseum MCP: +# - get_artwork_details +# - get_artwork_image +# - get_user_sets +# - get_user_set_details +# - open_image_in_browser +# - get_artist_timeline + +mcp_tools_filter = ["get_artwork_details", "get_artwork_image", "open_image_in_browser"] + + +def open_image_output_filter(output: str): + pattern = r"Successfully opened image in browser: " + text_to_print = re.sub(pattern, "", output) + print(f"🖼️ link to high resolution artwork: {text_to_print}") + + # We store functions so objects (e.g. SileroVADAnalyzer) don't get # instantiated. The function will be called when the desired transport gets # selected. @@ -136,7 +156,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # https://github.com/r-huijts/rijksmuseum-mcp args=["-y", "mcp-server-rijksmuseum"], env={"RIJKSMUSEUM_API_KEY": os.getenv("RIJKSMUSEUM_API_KEY")}, - ) + ), + # Optional + tools_filter=mcp_tools_filter, # Optional + tools_output_filters={"open_image_in_browser": open_image_output_filter}, ) except Exception as e: logger.error(f"error setting up mcp") @@ -155,7 +178,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. You have access to tools to search the Rijksmuseum collection. - Offer, for example, to show the earliest Rembrandt work from the museum. Use the `search_artwork` tool. + Offer, for example, to show a floral still life, use the `search_artwork` tool. The tool may respond with a JSON object with an `artworks` array. Choose the art from that array. Once the tool has responded, tell the user the title and use the `open_image_in_browser` tool. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. diff --git a/examples/foundational/39c-mcp-run-http.py b/examples/foundational/39a-mcp-streamable-http.py similarity index 100% rename from examples/foundational/39c-mcp-run-http.py rename to examples/foundational/39a-mcp-streamable-http.py diff --git a/examples/foundational/39d-mcp-run-http-gemini-live.py b/examples/foundational/39b-mcp-streamable-http-gemini-live.py similarity index 100% rename from examples/foundational/39d-mcp-run-http-gemini-live.py rename to examples/foundational/39b-mcp-streamable-http-gemini-live.py diff --git a/examples/foundational/39b-multiple-mcp.py b/examples/foundational/39c-multiple-mcp.py similarity index 79% rename from examples/foundational/39b-multiple-mcp.py rename to examples/foundational/39c-multiple-mcp.py index dad059208..e69f79f13 100644 --- a/examples/foundational/39b-multiple-mcp.py +++ b/examples/foundational/39c-multiple-mcp.py @@ -7,6 +7,7 @@ import asyncio import io +import json import os import re import shutil @@ -15,7 +16,7 @@ import aiohttp from dotenv import load_dotenv from loguru import logger from mcp import StdioServerParameters -from mcp.client.session_group import SseServerParameters +from mcp.client.session_group import StreamableHttpParameters from PIL import Image from pipecat.adapters.schemas.tools_schema import ToolsSchema @@ -66,11 +67,14 @@ class UrlToImageProcessor(FrameProcessor): await self.push_frame(frame, direction) def extract_url(self, text: str): - pattern = r"!\[[^\]]*\]\((https?://[^)]+\.(png|jpg|jpeg|PNG|JPG|JPEG|gif))\)" - match = re.search(pattern, text) - if match: - return match.group(1) - return None + try: + data = json.loads(text) + if "artObject" in data: + return data["artObject"]["webImage"]["url"] + if "artworks" in data and len(data["artworks"]): + return data["artworks"][0]["webImage"]["url"] + except: + pass async def run_image_process(self, image_url: str): try: @@ -132,10 +136,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): system = f""" You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. - You have access to tools to search the Rijksmuseum collection. - Offer, for example, to show the earliest Rembrandt work from the museum. Use the `search_artwork` tool. + You have access to tools to search the Rijksmuseum collection and the user's GitHub repositories and account. + Offer, for example, to show a floral still life, use the `search_artwork` tool. The tool may respond with a JSON object with an `artworks` array. Choose the art from that array. Once the tool has responded, tell the user the title and use the `open_image_in_browser` tool. + You can also offer to answer users questions about their GitHub repositories and account. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way. Don't overexplain what you are doing. @@ -145,11 +150,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [{"role": "system", "content": system}] try: - mcp = MCPClient( + rijksmuseum_mcp = MCPClient( server_params=StdioServerParameters( command=shutil.which("npx"), # https://github.com/r-huijts/rijksmuseum-mcp - args=["-y", "mcp-server-error setting up mcp"], + args=["-y", "mcp-server-rijksmuseum"], env={"RIJKSMUSEUM_API_KEY": os.getenv("RIJKSMUSEUM_API_KEY")}, ) ) @@ -157,24 +162,32 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.error(f"error setting up rijksmuseum mcp") logger.exception("error trace:") try: - # https://docs.mcp.run/integrating/tutorials/mcp-run-sse-openai-agents/ - # ie. "https://www.mcp.run/api/mcp/sse?..." - # ensure the profile has a tool or few installed - mcp_run = MCPClient(server_params=SseServerParameters(url=os.getenv("MCP_RUN_SSE_URL"))) + # Github MCP docs: https://github.com/github/github-mcp-server + # Enable Github Copilot on your GitHub account. Free tier is ok. (https://github.com/settings/copilot) + # Generate a personal access token. It must be a Fine-grained token, classic tokens are not supported. (https://github.com/settings/personal-access-tokens) + # Set permissions you want to use (eg. "all repositories", "profile: read/write", etc) + github_mcp = MCPClient( + server_params=StreamableHttpParameters( + url="https://api.githubcopilot.com/mcp/", + headers={ + "Authorization": f"Bearer {os.getenv('GITHUB_PERSONAL_ACCESS_TOKEN')}" + }, + ) + ) except Exception as e: logger.error(f"error setting up mcp.run") logger.exception("error trace:") - tools = {} - run_tools = {} + rijksmuseum_tools = {} + github_tools = {} try: - tools = await mcp.register_tools(llm) - run_tools = await mcp_run.register_tools(llm) + rijksmuseum_tools = await rijksmuseum_mcp.register_tools(llm) + github_tools = await github_mcp.register_tools(llm) except Exception as e: logger.error(f"error registering tools") logger.exception("error trace:") - all_standard_tools = run_tools.standard_tools + tools.standard_tools + all_standard_tools = rijksmuseum_tools.standard_tools + github_tools.standard_tools all_tools = ToolsSchema(standard_tools=all_standard_tools) context = LLMContext(messages, all_tools) @@ -226,9 +239,9 @@ async def bot(runner_args: RunnerArguments): if __name__ == "__main__": - if not os.getenv("RIJKSMUSEUM_API_KEY") or not os.getenv("MCP_RUN_SSE_URL"): + if not os.getenv("RIJKSMUSEUM_API_KEY") or not os.getenv("GITHUB_PERSONAL_ACCESS_TOKEN"): logger.error( - f"Please set RIJKSMUSEUM_API_KEY and MCP_RUN_SSE_URL environment variables. See https://github.com/r-huijts/rijksmuseum-mcp and https://mcp.run" + f"Please set `RIJKSMUSEUM_API_KEY` and `GITHUB_PERSONAL_ACCESS_TOKEN` environment variables. See https://github.com/r-huijts/rijksmuseum-mcp." ) import sys diff --git a/pyproject.toml b/pyproject.toml index e313c789d..bb72dba61 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,61 +45,63 @@ Source = "https://github.com/pipecat-ai/pipecat" Website = "https://pipecat.ai" [project.optional-dependencies] -aic = [ "aic-sdk~=1.1.0" ] +aic = [ "aic-sdk~=1.2.0" ] anthropic = [ "anthropic~=0.49.0" ] assemblyai = [ "pipecat-ai[websockets-base]" ] asyncai = [ "pipecat-ai[websockets-base]" ] -aws = [ "aioboto3~=15.0.0", "pipecat-ai[websockets-base]" ] -aws-nova-sonic = [ "aws_sdk_bedrock_runtime~=0.1.1; python_version>='3.12'" ] +aws = [ "aioboto3~=15.5.0", "pipecat-ai[websockets-base]" ] +aws-nova-sonic = [ "aws_sdk_bedrock_runtime~=0.2.0; python_version>='3.12'" ] azure = [ "azure-cognitiveservices-speech~=1.42.0"] cartesia = [ "cartesia~=2.0.3", "pipecat-ai[websockets-base]" ] cerebras = [] +daily = [ "daily-python~=0.22.0" ] +deepgram = [ "deepgram-sdk~=4.7.0", "pipecat-ai[websockets-base]" ] deepseek = [] -daily = [ "daily-python~=0.21.0" ] -deepgram = [ "deepgram-sdk~=4.7.0" ] elevenlabs = [ "pipecat-ai[websockets-base]" ] fal = [ "fal-client~=0.5.9" ] fireworks = [] fish = [ "ormsgpack~=1.7.0", "pipecat-ai[websockets-base]" ] gladia = [ "pipecat-ai[websockets-base]" ] google = [ "google-cloud-speech>=2.33.0,<3", "google-cloud-texttospeech>=2.31.0,<3", "google-genai>=1.41.0,<2", "pipecat-ai[websockets-base]" ] +gradium = [ "pipecat-ai[websockets-base]" ] grok = [] groq = [ "groq~=0.23.0" ] gstreamer = [ "pygobject~=3.50.0" ] heygen = [ "livekit>=1.0.13", "pipecat-ai[websockets-base]" ] hume = [ "hume>=0.11.2" ] inworld = [] -krisp = [ "pipecat-ai-krisp~=0.4.0" ] koala = [ "pvkoala~=2.0.3" ] +krisp = [ "pipecat-ai-krisp~=0.4.0" ] langchain = [ "langchain~=0.3.20", "langchain-community~=0.3.20", "langchain-openai~=0.3.9" ] -livekit = [ "livekit~=1.0.13", "livekit-api~=1.0.5", "tenacity>=8.2.3,<10.0.0" ] +livekit = [ "livekit~=1.0.13", "livekit-api~=1.0.5", "tenacity>=8.2.3,<10.0.0", "pyjwt>=2.10.1" ] lmnt = [ "pipecat-ai[websockets-base]" ] local = [ "pyaudio~=0.2.14" ] +local-smart-turn = [ "coremltools>=8.0", "transformers", "torch>=2.5.0,<3", "torchaudio>=2.5.0,<3" ] +local-smart-turn-v3 = [ "transformers", "onnxruntime>=1.20.1,<2" ] mcp = [ "mcp[cli]>=1.11.0,<2" ] mem0 = [ "mem0ai~=0.1.94" ] mistral = [] mlx-whisper = [ "mlx-whisper~=0.4.2" ] moondream = [ "accelerate~=1.10.0", "einops~=0.8.0", "pyvips[binary]~=3.0.0", "timm~=1.0.13", "transformers>=4.48.0" ] -nim = [] neuphonic = [ "pipecat-ai[websockets-base]" ] noisereduce = [ "noisereduce~=3.0.3" ] +nvidia = [ "nvidia-riva-client~=2.21.1" ] openai = [ "pipecat-ai[websockets-base]" ] openpipe = [ "openpipe>=4.50.0,<6" ] openrouter = [] perplexity = [] playht = [ "pipecat-ai[websockets-base]" ] qwen = [] +remote-smart-turn = [] rime = [ "pipecat-ai[websockets-base]" ] -riva = [ "nvidia-riva-client~=2.21.1" ] +riva = [ "pipecat-ai[nvidia]" ] runner = [ "python-dotenv>=1.0.0,<2.0.0", "uvicorn>=0.32.0,<1.0.0", "fastapi>=0.115.6,<0.122.0", "pipecat-ai-small-webrtc-prebuilt>=1.0.0"] +sagemaker = ["aws_sdk_sagemaker_runtime_http2; python_version>='3.12'"] sambanova = [] sarvam = [ "sarvamai==0.1.21", "pipecat-ai[websockets-base]" ] sentry = [ "sentry-sdk>=2.28.0,<3" ] -local-smart-turn = [ "coremltools>=8.0", "transformers", "torch>=2.5.0,<3", "torchaudio>=2.5.0,<3" ] -local-smart-turn-v3 = [ "transformers", "onnxruntime>=1.20.1,<2" ] -remote-smart-turn = [] silero = [ "onnxruntime>=1.20.1,<2" ] -simli = [ "simli-ai~=0.1.25"] +simli = [ "simli-ai~=1.0.3"] soniox = [ "pipecat-ai[websockets-base]" ] soundfile = [ "soundfile~=0.13.1" ] speechmatics = [ "speechmatics-rt>=0.5.0" ] @@ -128,6 +130,7 @@ dev = [ "setuptools~=78.1.1", "setuptools_scm~=8.3.1", "python-dotenv>=1.0.1,<2.0.0", + "towncrier~=25.8.0", ] docs = [ @@ -158,7 +161,7 @@ where = ["src"] "src/pipecat/audio/dtmf/dtmf-star.wav", ] "pipecat.services.aws_nova_sonic" = ["src/pipecat/services/aws_nova_sonic/ready.wav"] -"pipecat.audio.turn.smart_turn.data" = ["src/pipecat/audio/turn/smart_turn/data/smart-turn-v3.0.onnx"] +"pipecat.audio.turn.smart_turn.data" = ["src/pipecat/audio/turn/smart_turn/data/smart-turn-v3.1-cpu.onnx"] [tool.pytest.ini_options] addopts = "--verbose" @@ -205,3 +208,44 @@ convention = "google" command_line = "--module pytest" source = ["src"] omit = ["*/tests/*"] + +[tool.towncrier] +package = "pipecat" +package_dir = "src" +filename = "CHANGELOG.md" +directory = "changelog" +start_string = "\n" +template = "changelog/_template.md.j2" +title_format = "## [{version}] - {project_date}" +underlines = ["", "", ""] +wrap = true + +[[tool.towncrier.type]] +directory = "added" +name = "Added" +showcontent = true + +[[tool.towncrier.type]] +directory = "changed" +name = "Changed" +showcontent = true + +[[tool.towncrier.type]] +directory = "deprecated" +name = "Deprecated" +showcontent = true + +[[tool.towncrier.type]] +directory = "removed" +name = "Removed" +showcontent = true + +[[tool.towncrier.type]] +directory = "fixed" +name = "Fixed" +showcontent = true + +[[tool.towncrier.type]] +directory = "security" +name = "Security" +showcontent = true diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index da6df053d..f45128133 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -30,8 +30,8 @@ EVAL_SIMPLE_MATH = EvalConfig( ) EVAL_WEATHER = EvalConfig( - prompt="What's the weather in San Francisco?", - eval="The user says something specific about the current weather in San Francisco, including the degrees.", + prompt="What's the weather in San Francisco (in farhenheit or celsius)?", + eval="The user says something specific about the current weather in San Francisco, including the degrees (in farhenheit or celsius).", ) EVAL_ONLINE_SEARCH = EvalConfig( @@ -70,7 +70,7 @@ EVAL_VOICEMAIL = EvalConfig( EVAL_CONVERSATION = EvalConfig( prompt="Hello, this is Mark.", - eval="The user replies with a greeting.", + eval="The user acknowledges the greeting.", eval_speaks_first=True, ) @@ -103,7 +103,7 @@ TESTS_07 = [ ("07o-interruptible-assemblyai.py", EVAL_SIMPLE_MATH), ("07q-interruptible-rime.py", EVAL_SIMPLE_MATH), ("07q-interruptible-rime-http.py", EVAL_SIMPLE_MATH), - ("07r-interruptible-riva-nim.py", EVAL_SIMPLE_MATH), + ("07r-interruptible-nvidia.py", EVAL_SIMPLE_MATH), ("07s-interruptible-google-audio-in.py", EVAL_SIMPLE_MATH), ("07t-interruptible-fish.py", EVAL_SIMPLE_MATH), ("07v-interruptible-neuphonic.py", EVAL_SIMPLE_MATH), @@ -136,7 +136,7 @@ TESTS_14 = [ ("14g-function-calling-grok.py", EVAL_WEATHER), ("14h-function-calling-azure.py", EVAL_WEATHER), ("14i-function-calling-fireworks.py", EVAL_WEATHER), - ("14j-function-calling-nim.py", EVAL_WEATHER), + ("14j-function-calling-nvidia.py", EVAL_WEATHER), ("14k-function-calling-cerebras.py", EVAL_WEATHER), ("14m-function-calling-openrouter.py", EVAL_WEATHER), ("14n-function-calling-perplexity.py", EVAL_WEATHER), diff --git a/src/pipecat/audio/filters/aic_filter.py b/src/pipecat/audio/filters/aic_filter.py index 2f4699912..6d15fae7e 100644 --- a/src/pipecat/audio/filters/aic_filter.py +++ b/src/pipecat/audio/filters/aic_filter.py @@ -39,7 +39,7 @@ class AICFilter(BaseAudioFilter): self, *, license_key: str = "", - model_type: AICModelType = AICModelType.QUAIL_L, + model_type: AICModelType = AICModelType.QUAIL_STT, enhancement_level: Optional[float] = 1.0, voice_gain: Optional[float] = 1.0, noise_gate_enable: Optional[bool] = True, @@ -52,12 +52,27 @@ class AICFilter(BaseAudioFilter): enhancement_level: Optional overall enhancement strength (0.0..1.0). voice_gain: Optional linear gain applied to detected speech (0.0..4.0). noise_gate_enable: Optional enable/disable noise gate (default: True). + + .. deprecated:: 1.3.0 + The `noise_gate_enable` parameter is deprecated and no longer has any effect. + It will be removed in a future version. """ self._license_key = license_key self._model_type = model_type self._enhancement_level = enhancement_level self._voice_gain = voice_gain + if noise_gate_enable is not None: + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Parameter `noise_gate_enable` is deprecated and no longer has any effect. " + "It will be removed in a future version. Use AIC VAD instead (create_vad_analyzer()).", + DeprecationWarning, + ) + self._noise_gate_enable = noise_gate_enable self._enabled = True @@ -149,10 +164,6 @@ class AICFilter(BaseAudioFilter): ) if self._voice_gain is not None: self._aic.set_parameter(AICParameter.VOICE_GAIN, float(self._voice_gain)) - if self._noise_gate_enable is not None: - self._aic.set_parameter( - AICParameter.NOISE_GATE_ENABLE, 1.0 if bool(self._noise_gate_enable) else 0.0 - ) self._aic_ready = True diff --git a/src/pipecat/audio/turn/smart_turn/base_smart_turn.py b/src/pipecat/audio/turn/smart_turn/base_smart_turn.py index 4fba0f703..5cb832430 100644 --- a/src/pipecat/audio/turn/smart_turn/base_smart_turn.py +++ b/src/pipecat/audio/turn/smart_turn/base_smart_turn.py @@ -28,7 +28,6 @@ from pipecat.metrics.metrics import MetricsData, SmartTurnMetricsData STOP_SECS = 3 PRE_SPEECH_MS = 0 MAX_DURATION_SECONDS = 8 # Max allowed segment duration -USE_ONLY_LAST_VAD_SEGMENT = True class SmartTurnParams(BaseTurnParams): @@ -43,8 +42,6 @@ class SmartTurnParams(BaseTurnParams): stop_secs: float = STOP_SECS pre_speech_ms: float = PRE_SPEECH_MS max_duration_secs: float = MAX_DURATION_SECONDS - # not exposing this for now yet until the model can handle it. - # use_only_last_vad_segment: bool = USE_ONLY_LAST_VAD_SEGMENT class SmartTurnTimeoutException(Exception): @@ -160,7 +157,7 @@ class BaseSmartTurn(BaseTurnAnalyzer): state, result = await loop.run_in_executor( self._executor, self._process_speech_segment, self._audio_buffer ) - if state == EndOfTurnState.COMPLETE or USE_ONLY_LAST_VAD_SEGMENT: + if state == EndOfTurnState.COMPLETE: self._clear(state) logger.debug(f"End of Turn result: {state}") return state, result diff --git a/src/pipecat/audio/turn/smart_turn/data/smart-turn-v3.0.onnx b/src/pipecat/audio/turn/smart_turn/data/smart-turn-v3.1-cpu.onnx similarity index 56% rename from src/pipecat/audio/turn/smart_turn/data/smart-turn-v3.0.onnx rename to src/pipecat/audio/turn/smart_turn/data/smart-turn-v3.1-cpu.onnx index e2ff710a0..9a2360f19 100644 Binary files a/src/pipecat/audio/turn/smart_turn/data/smart-turn-v3.0.onnx and b/src/pipecat/audio/turn/smart_turn/data/smart-turn-v3.1-cpu.onnx differ diff --git a/src/pipecat/audio/turn/smart_turn/local_smart_turn_v3.py b/src/pipecat/audio/turn/smart_turn/local_smart_turn_v3.py index 08b9f3cd1..406abe165 100644 --- a/src/pipecat/audio/turn/smart_turn/local_smart_turn_v3.py +++ b/src/pipecat/audio/turn/smart_turn/local_smart_turn_v3.py @@ -42,17 +42,15 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn): Args: smart_turn_model_path: Path to the ONNX model file. If this is not - set, the bundled smart-turn-v3.0 model will be used. + set, the bundled smart-turn-v3.1-cpu model will be used. cpu_count: The number of CPUs to use for inference. Defaults to 1. **kwargs: Additional arguments passed to BaseSmartTurn. """ super().__init__(**kwargs) - logger.debug("Loading Local Smart Turn v3 model...") - if not smart_turn_model_path: # Load bundled model - model_name = "smart-turn-v3.0.onnx" + model_name = "smart-turn-v3.1-cpu.onnx" package_path = "pipecat.audio.turn.smart_turn.data" try: @@ -70,6 +68,8 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn): impresources.files(package_path).joinpath(model_name) ) + logger.debug(f"Loading Local Smart Turn v3.x model from {smart_turn_model_path}...") + so = ort.SessionOptions() so.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL so.inter_op_num_threads = 1 @@ -79,7 +79,7 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn): self._feature_extractor = WhisperFeatureExtractor(chunk_length=8) self._session = ort.InferenceSession(smart_turn_model_path, sess_options=so) - logger.debug("Loaded Local Smart Turn v3") + logger.debug("Loaded Local Smart Turn v3.x") def _predict_endpoint(self, audio_array: np.ndarray) -> Dict[str, Any]: """Predict end-of-turn using local ONNX model.""" diff --git a/src/pipecat/extensions/ivr/ivr_navigator.py b/src/pipecat/extensions/ivr/ivr_navigator.py index 05748d94f..505224cde 100644 --- a/src/pipecat/extensions/ivr/ivr_navigator.py +++ b/src/pipecat/extensions/ivr/ivr_navigator.py @@ -18,8 +18,10 @@ from loguru import logger from pipecat.audio.dtmf.types import KeypadEntry from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import ( + EndFrame, Frame, LLMContextFrame, + LLMFullResponseEndFrame, LLMMessagesUpdateFrame, LLMTextFrame, OutputDTMFUrgentFrame, @@ -31,7 +33,11 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.llm_service import LLMService -from pipecat.utils.text.pattern_pair_aggregator import PatternMatch, PatternPairAggregator +from pipecat.utils.text.pattern_pair_aggregator import ( + MatchAction, + PatternMatch, + PatternPairAggregator, +) class IVRStatus(Enum): @@ -114,15 +120,15 @@ class IVRProcessor(FrameProcessor): def _setup_xml_patterns(self): """Set up XML pattern detection and handlers.""" # Register DTMF pattern - self._aggregator.add_pattern_pair("dtmf", "", "", remove_match=True) + self._aggregator.add_pattern("dtmf", "", "", action=MatchAction.REMOVE) self._aggregator.on_pattern_match("dtmf", self._handle_dtmf_action) # Register mode pattern - self._aggregator.add_pattern_pair("mode", "", "", remove_match=True) + self._aggregator.add_pattern("mode", "", "", action=MatchAction.REMOVE) self._aggregator.on_pattern_match("mode", self._handle_mode_action) # Register IVR pattern - self._aggregator.add_pattern_pair("ivr", "", "", remove_match=True) + self._aggregator.add_pattern("ivr", "", "", action=MatchAction.REMOVE) self._aggregator.on_pattern_match("ivr", self._handle_ivr_action) async def process_frame(self, frame: Frame, direction: FrameDirection): @@ -145,10 +151,17 @@ class IVRProcessor(FrameProcessor): elif isinstance(frame, LLMTextFrame): # Process text through the pattern aggregator - result = await self._aggregator.aggregate(frame.text) - if result: + async for result in self._aggregator.aggregate(frame.text): # Push aggregated text that doesn't contain XML patterns - await self.push_frame(LLMTextFrame(result), direction) + await self.push_frame(LLMTextFrame(result.text), direction) + + elif isinstance(frame, (LLMFullResponseEndFrame, EndFrame)): + # Flush any remaining text from the aggregator + remaining = await self._aggregator.flush() + if remaining: + await self.push_frame(LLMTextFrame(remaining.text), direction) + # Push the end frame + await self.push_frame(frame, direction) else: await self.push_frame(frame, direction) @@ -159,7 +172,7 @@ class IVRProcessor(FrameProcessor): Args: match: The pattern match containing DTMF content. """ - value = match.content + value = match.text logger.debug(f"DTMF detected: {value}") try: @@ -180,7 +193,7 @@ class IVRProcessor(FrameProcessor): Args: match: The pattern match containing IVR status content. """ - status = match.content + status = match.text logger.trace(f"IVR status detected: {status}") # Convert string to enum, with validation @@ -211,7 +224,7 @@ class IVRProcessor(FrameProcessor): Args: match: The pattern match containing mode content. """ - mode = match.content + mode = match.text logger.debug(f"Mode detected: {mode}") if mode == "conversation": await self._handle_conversation() diff --git a/src/pipecat/extensions/voicemail/voicemail_detector.py b/src/pipecat/extensions/voicemail/voicemail_detector.py index 149103319..e79520cb5 100644 --- a/src/pipecat/extensions/voicemail/voicemail_detector.py +++ b/src/pipecat/extensions/voicemail/voicemail_detector.py @@ -40,8 +40,8 @@ from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, FrameProcessorSetup from pipecat.services.llm_service import LLMService -from pipecat.sync.base_notifier import BaseNotifier -from pipecat.sync.event_notifier import EventNotifier +from pipecat.utils.sync.base_notifier import BaseNotifier +from pipecat.utils.sync.event_notifier import EventNotifier class NotifierGate(FrameProcessor): diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index ddb4e5a14..9cb969f28 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -12,6 +12,7 @@ and LLM processing. """ from dataclasses import dataclass, field +from enum import Enum from typing import ( TYPE_CHECKING, Any, @@ -185,6 +186,20 @@ class ControlFrame(Frame): # +@dataclass +class UninterruptibleFrame: + """A marker for data or control frames that must not be interrupted. + + Frames with this mixin are still ordered normally, but unlike other frames, + they are preserved during interruptions: they remain in internal queues and + any task processing them will not be cancelled. This ensures the frame is + always delivered and processed to completion. + + """ + + pass + + @dataclass class AudioRawFrame: """A frame containing a chunk of raw audio. @@ -329,7 +344,7 @@ class TextFrame(DataFrame): """ text: str - skip_tts: bool = field(init=False) + skip_tts: Optional[bool] = field(init=False) # Whether any necessary inter-frame (leading/trailing) spaces are already # included in the text. # NOTE: Ideally this would be available at init time with a default value, @@ -337,11 +352,14 @@ class TextFrame(DataFrame): # mandatory fields of theirs to have defaults to preserve # non-default-before-default argument order) includes_inter_frame_spaces: bool = field(init=False) + # Whether this text frame should be appended to the LLM context. + append_to_context: bool = field(init=False) def __post_init__(self): super().__post_init__() - self.skip_tts = False + self.skip_tts = None self.includes_inter_frame_spaces = False + self.append_to_context = True def __str__(self): pts = format_pts(self.pts) @@ -358,8 +376,32 @@ class LLMTextFrame(TextFrame): self.includes_inter_frame_spaces = True +class AggregationType(str, Enum): + """Built-in aggregation strings.""" + + SENTENCE = "sentence" + WORD = "word" + + def __str__(self): + return self.value + + @dataclass -class TTSTextFrame(TextFrame): +class AggregatedTextFrame(TextFrame): + """Text frame representing an aggregation of TextFrames. + + This frame contains multiple TextFrames aggregated together for processing + or output along with a field to indicate how they are aggregated. + + Parameters: + aggregated_by: Method used to aggregate the text frames. + """ + + aggregated_by: AggregationType | str + + +@dataclass +class TTSTextFrame(AggregatedTextFrame): """Text frame generated by Text-to-Speech services.""" pass @@ -668,6 +710,44 @@ class LLMConfigureOutputFrame(DataFrame): skip_tts: bool +@dataclass +class FunctionCallResultProperties: + """Properties for configuring function call result behavior. + + Parameters: + run_llm: Whether to run the LLM after receiving this result. + on_context_updated: Callback to execute when context is updated. + """ + + run_llm: Optional[bool] = None + on_context_updated: Optional[Callable[[], Awaitable[None]]] = None + + +@dataclass +class FunctionCallResultFrame(DataFrame, UninterruptibleFrame): + """Frame containing the result of an LLM function call. + + This is an uninterruptible frame because once a result is generated we + always want to update the context. + + Parameters: + function_name: Name of the function that was executed. + tool_call_id: Unique identifier for the function call. + arguments: Arguments that were passed to the function. + result: The result returned by the function. + run_llm: Whether to run the LLM after this result. + properties: Additional properties for result handling. + + """ + + function_name: str + tool_call_id: str + arguments: Any + result: Any + run_llm: Optional[bool] = None + properties: Optional[FunctionCallResultProperties] = None + + @dataclass class TTSSpeakFrame(DataFrame): """Frame containing text that should be spoken by TTS. @@ -807,11 +887,13 @@ class ErrorFrame(SystemFrame): error: Description of the error that occurred. fatal: Whether the error is fatal and requires bot shutdown. processor: The frame processor that generated the error. + exception: The exception that occurred. """ error: str fatal: bool = False processor: Optional["FrameProcessor"] = None + exception: Optional[Exception] = None def __str__(self): return f"{self.name}(error: {self.error}, fatal: {self.fatal})" @@ -1059,23 +1141,6 @@ class FunctionCallsStartedFrame(SystemFrame): function_calls: Sequence[FunctionCallFromLLM] -@dataclass -class FunctionCallInProgressFrame(SystemFrame): - """Frame signaling that a function call is currently executing. - - Parameters: - function_name: Name of the function being executed. - tool_call_id: Unique identifier for this function call. - arguments: Arguments passed to the function. - cancel_on_interruption: Whether to cancel this call if interrupted. - """ - - function_name: str - tool_call_id: str - arguments: Any - cancel_on_interruption: bool = False - - @dataclass class FunctionCallCancelFrame(SystemFrame): """Frame signaling that a function call has been cancelled. @@ -1089,40 +1154,6 @@ class FunctionCallCancelFrame(SystemFrame): tool_call_id: str -@dataclass -class FunctionCallResultProperties: - """Properties for configuring function call result behavior. - - Parameters: - run_llm: Whether to run the LLM after receiving this result. - on_context_updated: Callback to execute when context is updated. - """ - - run_llm: Optional[bool] = None - on_context_updated: Optional[Callable[[], Awaitable[None]]] = None - - -@dataclass -class FunctionCallResultFrame(SystemFrame): - """Frame containing the result of an LLM function call. - - Parameters: - function_name: Name of the function that was executed. - tool_call_id: Unique identifier for the function call. - arguments: Arguments that were passed to the function. - result: The result returned by the function. - run_llm: Whether to run the LLM after this result. - properties: Additional properties for result handling. - """ - - function_name: str - tool_call_id: str - arguments: Any - result: Any - run_llm: Optional[bool] = None - properties: Optional[FunctionCallResultProperties] = None - - @dataclass class STTMuteFrame(SystemFrame): """Frame to mute/unmute the Speech-to-Text service. @@ -1602,22 +1633,43 @@ class LLMFullResponseStartFrame(ControlFrame): more TextFrames and a final LLMFullResponseEndFrame. """ - skip_tts: bool = field(init=False) + skip_tts: Optional[bool] = field(init=False) def __post_init__(self): super().__post_init__() - self.skip_tts = False + self.skip_tts = None @dataclass class LLMFullResponseEndFrame(ControlFrame): """Frame indicating the end of an LLM response.""" - skip_tts: bool = field(init=False) + skip_tts: Optional[bool] = field(init=False) def __post_init__(self): super().__post_init__() - self.skip_tts = False + self.skip_tts = None + + +@dataclass +class FunctionCallInProgressFrame(ControlFrame, UninterruptibleFrame): + """Frame signaling that a function call is currently executing. + + This is an uninterruptible frame because we always want to update the + context. + + Parameters: + function_name: Name of the function being executed. + tool_call_id: Unique identifier for this function call. + arguments: Arguments passed to the function. + cancel_on_interruption: Whether to cancel this call if interrupted. + + """ + + function_name: str + tool_call_id: str + arguments: Any + cancel_on_interruption: bool = False @dataclass diff --git a/src/pipecat/processors/aggregators/gated_llm_context.py b/src/pipecat/processors/aggregators/gated_llm_context.py index 6535bffbb..9c3d90948 100644 --- a/src/pipecat/processors/aggregators/gated_llm_context.py +++ b/src/pipecat/processors/aggregators/gated_llm_context.py @@ -9,7 +9,7 @@ from pipecat.frames.frames import CancelFrame, EndFrame, Frame, LLMContextFrame, StartFrame from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.sync.base_notifier import BaseNotifier +from pipecat.utils.sync.base_notifier import BaseNotifier class GatedLLMContextAggregator(FrameProcessor): diff --git a/src/pipecat/processors/aggregators/llm_context.py b/src/pipecat/processors/aggregators/llm_context.py index b9216103a..99b9aeaa9 100644 --- a/src/pipecat/processors/aggregators/llm_context.py +++ b/src/pipecat/processors/aggregators/llm_context.py @@ -180,7 +180,7 @@ class LLMContext: text: Optional text to include with the audio. """ - def encode_audio(): + async def encode_audio(): sample_rate = audio_frames[0].sample_rate num_channels = audio_frames[0].num_channels @@ -198,7 +198,7 @@ class LLMContext: encoded_audio = base64.b64encode(buffer.getvalue()).decode("utf-8") return encoded_audio - encoded_audio = asyncio.to_thread(encode_audio) + encoded_audio = await asyncio.to_thread(encode_audio) content.append( { @@ -333,7 +333,7 @@ class LLMContext: """ self._tool_choice = tool_choice - def add_image_frame_message( + async def add_image_frame_message( self, *, format: str, size: tuple[int, int], image: bytes, text: Optional[str] = None ): """Add a message containing an image frame. @@ -344,10 +344,12 @@ class LLMContext: image: Raw image bytes. text: Optional text to include with the image. """ - message = LLMContext.create_image_message(format=format, size=size, image=image, text=text) + message = await LLMContext.create_image_message( + format=format, size=size, image=image, text=text + ) self.add_message(message) - def add_audio_frames_message( + async def add_audio_frames_message( self, *, audio_frames: list[AudioRawFrame], text: str = "Audio follows" ): """Add a message containing audio frames. @@ -356,7 +358,7 @@ class LLMContext: audio_frames: List of audio frame objects to include. text: Optional text to include with the audio. """ - message = LLMContext.create_audio_message(audio_frames=audio_frames, text=text) + message = await LLMContext.create_audio_message(audio_frames=audio_frames, text=text) self.add_message(message) @staticmethod diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index afc091d5a..ec13b643f 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -1001,7 +1001,7 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): await self.push_aggregation() async def _handle_text(self, frame: TextFrame): - if not self._started: + if not self._started or not frame.append_to_context: return if self._params.expect_stripped_words: diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index 8c084f2dc..69fc649ce 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -591,8 +591,6 @@ class LLMAssistantAggregator(LLMContextAggregator): self._started = 0 self._function_calls_in_progress: Dict[str, Optional[FunctionCallInProgressFrame]] = {} self._context_updated_tasks: Set[asyncio.Task] = set() - self._function_calls_context_messages = [] - self._function_calls_pending_context_updates_callbacks = [] @property def has_function_calls_in_progress(self) -> bool: @@ -649,23 +647,21 @@ class LLMAssistantAggregator(LLMContextAggregator): async def push_aggregation(self): """Push the current assistant aggregation with timestamp.""" - if self._aggregation: - aggregation = self.aggregation_string() - await self.reset() + if not self._aggregation: + return - if aggregation: - self._context.add_message({"role": "assistant", "content": aggregation}) + aggregation = self.aggregation_string() + await self.reset() - # Push context frame - await self.push_context_frame() + if aggregation: + self._context.add_message({"role": "assistant", "content": aggregation}) - # Push timestamp frame with current time - timestamp_frame = LLMContextAssistantTimestampFrame(timestamp=time_now_iso8601()) - await self.push_frame(timestamp_frame) + # Push context frame + await self.push_context_frame() - if self._function_calls_context_messages: - self._flush_function_call_messages_to_context() - await self.push_context_frame(FrameDirection.UPSTREAM) + # Push timestamp frame with current time + timestamp_frame = LLMContextAssistantTimestampFrame(timestamp=time_now_iso8601()) + await self.push_frame(timestamp_frame) async def _handle_llm_run(self, frame: LLMRunFrame): await self.push_context_frame(FrameDirection.UPSTREAM) @@ -685,23 +681,6 @@ class LLMAssistantAggregator(LLMContextAggregator): self._started = 0 await self.reset() - def _flush_function_call_messages_to_context(self): - """Move all function calls messages into context, then clear the list.""" - if self._function_calls_context_messages: - self._context.add_messages(self._function_calls_context_messages) - self._function_calls_context_messages.clear() - - # Call the `on_context_updated` callbacks once the function call results - # are added to the context. Run them in separate tasks to make - # sure we don't block the pipeline. - for callback, task_name in self._function_calls_pending_context_updates_callbacks: - task = self.create_task(callback(), task_name) - self._context_updated_tasks.add(task) - task.add_done_callback(self._context_updated_task_finished) - - # Clear the pending callbacks list - self._function_calls_pending_context_updates_callbacks.clear() - async def _handle_function_calls_started(self, frame: FunctionCallsStartedFrame): function_names = [f"{f.function_name}:{f.tool_call_id}" for f in frame.function_calls] logger.debug(f"{self} FunctionCallsStartedFrame: {function_names}") @@ -714,7 +693,7 @@ class LLMAssistantAggregator(LLMContextAggregator): ) # Update context with the in-progress function call - self._function_calls_context_messages.append( + self._context.add_message( { "role": "assistant", "tool_calls": [ @@ -729,7 +708,7 @@ class LLMAssistantAggregator(LLMContextAggregator): ], } ) - self._function_calls_context_messages.append( + self._context.add_message( { "role": "tool", "content": "IN_PROGRESS", @@ -760,13 +739,6 @@ class LLMAssistantAggregator(LLMContextAggregator): else: self._update_function_call_result(frame.function_name, frame.tool_call_id, "COMPLETED") - # Store the on_context_updated callback along with task name info to be invoked later - if properties and properties.on_context_updated: - task_name = f"{frame.function_name}:{frame.tool_call_id}:on_context_updated" - self._function_calls_pending_context_updates_callbacks.append( - (properties.on_context_updated, task_name) - ) - run_llm = False # Run inference if the function call result requires it. @@ -781,13 +753,17 @@ class LLMAssistantAggregator(LLMContextAggregator): # If this is the last function call in progress, run the LLM. run_llm = not bool(self._function_calls_in_progress) - # Only run if the LLM response has completed (not currently generating), - # otherwise defer execution until push_aggregation() is called - # (triggered by LLMFullResponseEndFrame or interruption). - if not self._started: - self._flush_function_call_messages_to_context() - if run_llm: - await self.push_context_frame(FrameDirection.UPSTREAM) + if run_llm: + await self.push_context_frame(FrameDirection.UPSTREAM) + + # Call the `on_context_updated` callback once the function call result + # is added to the context. Also, run this in a separate task to make + # sure we don't block the pipeline. + if properties and properties.on_context_updated: + task_name = f"{frame.function_name}:{frame.tool_call_id}:on_context_updated" + task = self.create_task(properties.on_context_updated(), task_name) + self._context_updated_tasks.add(task) + task.add_done_callback(self._context_updated_task_finished) async def _handle_function_call_cancel(self, frame: FunctionCallCancelFrame): logger.debug( @@ -802,12 +778,7 @@ class LLMAssistantAggregator(LLMContextAggregator): del self._function_calls_in_progress[frame.tool_call_id] def _update_function_call_result(self, function_name: str, tool_call_id: str, result: Any): - def iter_all(): - yield from self._function_calls_context_messages - # In case on long-running function call, the function may already be added to the context - yield from self._context.get_messages() - - for message in iter_all(): + for message in self._context.get_messages(): if ( not isinstance(message, LLMSpecificMessage) and message["role"] == "tool" @@ -822,7 +793,7 @@ class LLMAssistantAggregator(LLMContextAggregator): logger.debug(f"{self} Appending UserImageRawFrame to LLM context (size: {frame.size})") - self._context.add_image_frame_message( + await self._context.add_image_frame_message( format=frame.format, size=frame.size, image=frame.image, @@ -840,7 +811,7 @@ class LLMAssistantAggregator(LLMContextAggregator): await self.push_aggregation() async def _handle_text(self, frame: TextFrame): - if not self._started: + if not self._started or not frame.append_to_context: return # Make sure we really have text (spaces count, too!) diff --git a/src/pipecat/processors/aggregators/llm_text_processor.py b/src/pipecat/processors/aggregators/llm_text_processor.py new file mode 100644 index 000000000..f0de46617 --- /dev/null +++ b/src/pipecat/processors/aggregators/llm_text_processor.py @@ -0,0 +1,103 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""LLM text processor module for processing and aggregating raw LLM output text. + +This processor will convert LLMTextFrames into AggregatedTextFrames based on the +configured text aggregator. Using the customizable aggregator, it provides +functionality to handle or manipulate LLM text frames before they are sent to other +components such as TTS services or context aggregators. It can be used to pre-aggregate +and categorize, modify, or filter direct output tokens from the LLM. +""" + +from typing import Optional + +from pipecat.frames.frames import ( + AggregatedTextFrame, + EndFrame, + Frame, + InterruptionFrame, + LLMFullResponseEndFrame, + LLMTextFrame, +) +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.utils.text.base_text_aggregator import BaseTextAggregator +from pipecat.utils.text.simple_text_aggregator import SimpleTextAggregator + + +class LLMTextProcessor(FrameProcessor): + """A processor for handling or manipulating LLM text frames before they are processed further. + + This processor will convert LLMTextFrames into AggregatedTextFrames based on the configured + text aggregator. Using the customizable aggregator, it provides functionality to handle or + manipulate LLM text frames before they are sent to other components such as TTS services or + context aggregators. It can be used to pre-aggregate and categorize, modify, or filter direct + output tokens from the LLM. + """ + + def __init__(self, *, text_aggregator: Optional[BaseTextAggregator] = None, **kwargs): + """Initialize the LLM text processor. + + Args: + text_aggregator: An optional text aggregator to use for processing LLM text frames. By + default, a SimpleTextAggregator aggregating by sentence will be used. + **kwargs: Additional arguments passed to parent class. + + TODO: Allow transformations per aggregation type or all (and deprecate the TTS filters). + """ + super().__init__(**kwargs) + self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator() + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process an LLMTextFrames using the aggregator to generate AggregatedTextFrames. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ + await super().process_frame(frame, direction) + + if isinstance(frame, InterruptionFrame): + await self._handle_interruption(frame) + await self.push_frame(frame, direction) + elif isinstance(frame, LLMTextFrame): + await self._handle_llm_text(frame) + elif isinstance(frame, LLMFullResponseEndFrame): + await self._handle_llm_end(frame.skip_tts) + await self.push_frame(frame, direction) + elif isinstance(frame, EndFrame): + await self._handle_llm_end() + await self.push_frame(frame, direction) + else: + await self.push_frame(frame, direction) + + async def _handle_interruption(self, _): + """Handle interruptions by resetting the text aggregator.""" + await self._text_aggregator.handle_interruption() + + async def reset(self): + """Reset the internal state of the text processor and its aggregator.""" + await self._text_aggregator.reset() + + async def _handle_llm_text(self, in_frame: LLMTextFrame): + async for aggregation in self._text_aggregator.aggregate(in_frame.text): + out_frame = AggregatedTextFrame( + text=aggregation.text, + aggregated_by=aggregation.type, + ) + out_frame.skip_tts = in_frame.skip_tts + await self.push_frame(out_frame) + + async def _handle_llm_end(self, skip_tts: Optional[bool] = None): + # Flush any remaining text + remaining = await self._text_aggregator.flush() + if remaining: + out_frame = AggregatedTextFrame( + text=remaining.text, + aggregated_by=remaining.type, + ) + out_frame.skip_tts = skip_tts + await self.push_frame(out_frame) diff --git a/src/pipecat/processors/filters/wake_check_filter.py b/src/pipecat/processors/filters/wake_check_filter.py index 1f01d0a1b..3fe960b55 100644 --- a/src/pipecat/processors/filters/wake_check_filter.py +++ b/src/pipecat/processors/filters/wake_check_filter.py @@ -126,6 +126,4 @@ class WakeCheckFilter(FrameProcessor): else: await self.push_frame(frame, direction) except Exception as e: - error_msg = f"Error in wake word filter: {e}" - logger.exception(error_msg) - await self.push_error(ErrorFrame(error_msg)) + await self.push_error(error_msg=f"Error in wake word filter: {e}", exception=e) diff --git a/src/pipecat/processors/filters/wake_notifier_filter.py b/src/pipecat/processors/filters/wake_notifier_filter.py index c30f3b5d3..8ed3bc8c4 100644 --- a/src/pipecat/processors/filters/wake_notifier_filter.py +++ b/src/pipecat/processors/filters/wake_notifier_filter.py @@ -10,7 +10,7 @@ from typing import Awaitable, Callable, Tuple, Type from pipecat.frames.frames import Frame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.sync.base_notifier import BaseNotifier +from pipecat.utils.sync.base_notifier import BaseNotifier class WakeNotifierFilter(FrameProcessor): diff --git a/src/pipecat/processors/frame_processor.py b/src/pipecat/processors/frame_processor.py index 69aee0acd..ed77506dd 100644 --- a/src/pipecat/processors/frame_processor.py +++ b/src/pipecat/processors/frame_processor.py @@ -12,6 +12,7 @@ management, and frame flow control mechanisms. """ import asyncio +import traceback from dataclasses import dataclass from enum import Enum from typing import Any, Awaitable, Callable, Coroutine, List, Optional, Sequence, Tuple, Type @@ -32,6 +33,7 @@ from pipecat.frames.frames import ( InterruptionTaskFrame, StartFrame, SystemFrame, + UninterruptibleFrame, ) from pipecat.metrics.metrics import LLMTokenUsage, MetricsData from pipecat.observers.base_observer import BaseObserver, FrameProcessed, FramePushed @@ -142,6 +144,7 @@ class FrameProcessor(BaseObject): - on_after_process_frame: Called after a frame is processed - on_before_push_frame: Called before a frame is pushed - on_after_push_frame: Called after a frame is pushed + - on_error: Called when an error is raised in the frame processing. """ def __init__( @@ -209,6 +212,7 @@ class FrameProcessor(BaseObject): # The input task that handles all types of frames. It processes system # frames right away and queues non-system frames for later processing. self.__should_block_system_frames = False + self.__input_queue = FrameProcessorQueue() self.__input_event: Optional[asyncio.Event] = None self.__input_frame_task: Optional[asyncio.Task] = None @@ -218,8 +222,10 @@ class FrameProcessor(BaseObject): # called. To resume processing frames we need to call # `resume_processing_frames()` which will wake up the event. self.__should_block_frames = False + self.__process_queue = asyncio.Queue() self.__process_event: Optional[asyncio.Event] = None self.__process_frame_task: Optional[asyncio.Task] = None + self.__process_current_frame: Optional[Frame] = None # To interrupt a pipeline, we push an `InterruptionTaskFrame` upstream. # Then we wait for the corresponding `InterruptionFrame` to travel from @@ -234,6 +240,7 @@ class FrameProcessor(BaseObject): self._register_event_handler("on_after_process_frame", sync=True) self._register_event_handler("on_before_push_frame", sync=True) self._register_event_handler("on_after_push_frame", sync=True) + self._register_event_handler("on_error", sync=True) @property def id(self) -> int: @@ -630,7 +637,43 @@ class FrameProcessor(BaseObject): elif isinstance(frame, (FrameProcessorResumeFrame, FrameProcessorResumeUrgentFrame)): await self.__resume(frame) - async def push_error(self, error: ErrorFrame): + async def push_error( + self, + error_msg: str, + exception: Optional[Exception] = None, + fatal: bool = False, + ): + """Creates and pushes an ErrorFrame upstream. + + Creates and pushes an ErrorFrame upstream to notify other processors in the + pipeline about an error condition. The error frame will include context about + which processor generated the error. + + Args: + error_msg: Descriptive message explaining the error condition. + exception: Optional exception object that caused the error, if available. + This provides additional context for debugging and error handling. + fatal: Whether this error should be considered fatal to the pipeline. + Fatal errors typically cause the entire pipeline to stop processing. + Defaults to False for non-fatal errors. + + Example:: + + ```python + # Non-fatal error + await self.push_error("Failed to process audio chunk, skipping") + + # Fatal error with exception context + try: + result = some_critical_operation() + except Exception as e: + await self.push_error("Critical operation failed", exception=e, fatal=True) + ``` + """ + error_frame = ErrorFrame(error=error_msg, fatal=fatal, exception=exception, processor=self) + await self.push_error_frame(error=error_frame) + + async def push_error_frame(self, error: ErrorFrame): """Push an error frame upstream. Args: @@ -638,6 +681,18 @@ class FrameProcessor(BaseObject): """ if not error.processor: error.processor = self + await self._call_event_handler("on_error", error) + + if error.exception: + tb = traceback.extract_tb(error.exception.__traceback__) + last = tb[-1] + error_message = ( + f"{error.processor} exception ({last.filename}:{last.lineno}): {error.error}" + ) + else: + error_message = f"{error.processor} error: {error.error}" + + logger.error(error_message) await self.push_frame(error, FrameDirection.UPSTREAM) async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): @@ -754,13 +809,19 @@ class FrameProcessor(BaseObject): # interruption). Instead we just drain the queue because this is # an interruption. self.__reset_process_task() + elif isinstance(self.__process_current_frame, UninterruptibleFrame): + # We don't want to cancel UninterruptibleFrame, so we simply + # cleanup the queue. + self.__reset_process_queue() else: - # Cancel and re-create the process task including the queue. + # Cancel and re-create the process task. await self.__cancel_process_task() self.__create_process_task() except Exception as e: - logger.exception(f"Uncaught exception in {self} when handling _start_interruption: {e}") - await self.push_error(ErrorFrame(str(e))) + await self.push_error( + error_msg=f"Uncaught exception handling _start_interruption: {e}", + exception=e, + ) async def __internal_push_frame(self, frame: Frame, direction: FrameDirection): """Internal method to push frames to adjacent processors. @@ -797,8 +858,7 @@ class FrameProcessor(BaseObject): await self._observer.on_push_frame(data) await self._prev.queue_frame(frame, direction) except Exception as e: - logger.exception(f"Uncaught exception in {self}: {e}") - await self.push_error(ErrorFrame(str(e))) + await self.push_error(error_msg=f"Uncaught exception: {e}", exception=e) def _check_started(self, frame: Frame): """Check if the processor has been started. @@ -820,7 +880,6 @@ class FrameProcessor(BaseObject): if not self.__input_frame_task: self.__input_event = asyncio.Event() - self.__input_queue = FrameProcessorQueue() self.__input_frame_task = self.create_task(self.__input_frame_task_handler()) async def __cancel_input_task(self): @@ -838,9 +897,7 @@ class FrameProcessor(BaseObject): return if not self.__process_frame_task: - self.__should_block_frames = False - self.__process_event = asyncio.Event() - self.__process_queue = asyncio.Queue() + self.__reset_process_task() self.__process_frame_task = self.create_task(self.__process_frame_task_handler()) def __reset_process_task(self): @@ -850,10 +907,26 @@ class FrameProcessor(BaseObject): self.__should_block_frames = False self.__process_event = asyncio.Event() + self.__reset_process_queue() + + def __reset_process_queue(self): + """Reset non-system frame processing queue.""" + # Create a new queue to insert UninterruptibleFrame frames. + new_queue = asyncio.Queue() + + # Process current queue and keep UninterruptibleFrame frames. while not self.__process_queue.empty(): - self.__process_queue.get_nowait() + item = self.__process_queue.get_nowait() + if isinstance(item, UninterruptibleFrame): + new_queue.put_nowait(item) self.__process_queue.task_done() + # Put back UninterruptibleFrame frames into our process queue. + while not new_queue.empty(): + item = new_queue.get_nowait() + self.__process_queue.put_nowait(item) + new_queue.task_done() + async def __cancel_process_task(self): """Cancel the non-system frame processing task.""" if self.__process_frame_task: @@ -874,8 +947,7 @@ class FrameProcessor(BaseObject): await self._call_event_handler("on_after_process_frame", frame) except Exception as e: - logger.exception(f"{self}: error processing frame: {e}") - await self.push_error(ErrorFrame(str(e))) + await self.push_error(error_msg=f"Error processing frame: {e}", exception=e) async def __input_frame_task_handler(self): """Handle frames from the input queue. @@ -908,8 +980,12 @@ class FrameProcessor(BaseObject): async def __process_frame_task_handler(self): """Handle non-system frames from the process queue.""" while True: + self.__process_current_frame = None + (frame, direction, callback) = await self.__process_queue.get() + self.__process_current_frame = frame + if self.__should_block_frames and self.__process_event: logger.trace(f"{self}: frame processing paused") await self.__process_event.wait() diff --git a/src/pipecat/processors/frameworks/langchain.py b/src/pipecat/processors/frameworks/langchain.py index b8a472a3d..08a84760f 100644 --- a/src/pipecat/processors/frameworks/langchain.py +++ b/src/pipecat/processors/frameworks/langchain.py @@ -24,7 +24,7 @@ try: from langchain_core.messages import AIMessageChunk from langchain_core.runnables import Runnable except ModuleNotFoundError as e: - logger.exception("In order to use Langchain, you need to `pip install pipecat-ai[langchain]`. ") + logger.error("In order to use Langchain, you need to `pip install pipecat-ai[langchain]`. ") raise Exception(f"Missing module: {e}") @@ -113,6 +113,6 @@ class LangchainProcessor(FrameProcessor): except GeneratorExit: logger.warning(f"{self} generator was closed prematurely") except Exception as e: - logger.exception(f"{self} an unknown error occurred: {e}") + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) finally: await self.push_frame(LLMFullResponseEndFrame()) diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index f04cbd395..28e87ef2c 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -24,6 +24,7 @@ from typing import ( Literal, Mapping, Optional, + Tuple, Union, ) @@ -32,6 +33,8 @@ from pydantic import BaseModel, Field, PrivateAttr, ValidationError from pipecat.audio.utils import calculate_audio_volume from pipecat.frames.frames import ( + AggregatedTextFrame, + AggregationType, BotStartedSpeakingFrame, BotStoppedSpeakingFrame, CancelFrame, @@ -704,6 +707,29 @@ class RTVITextMessageData(BaseModel): text: str +class RTVIBotOutputMessageData(RTVITextMessageData): + """Data for bot output RTVI messages. + + Extends RTVITextMessageData to include metadata about the output. + """ + + spoken: bool = False # Indicates if the text has been spoken by TTS + aggregated_by: AggregationType | str + # Indicates what form the text is in (e.g., by word, sentence, etc.) + + +class RTVIBotOutputMessage(BaseModel): + """Message containing bot output text. + + An event meant to holistically represent what the bot is outputting, + along with metadata about the output and if it has been spoken. + """ + + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL + type: Literal["bot-output"] = "bot-output" + data: RTVIBotOutputMessageData + + class RTVIBotTranscriptionMessage(BaseModel): """Message containing bot transcription text. @@ -896,6 +922,7 @@ class RTVIObserverParams: Parameter `errors_enabled` is deprecated. Error messages are always enabled. Parameters: + bot_output_enabled: Indicates if bot output messages should be sent. bot_llm_enabled: Indicates if the bot's LLM messages should be sent. bot_tts_enabled: Indicates if the bot's TTS messages should be sent. bot_speaking_enabled: Indicates if the bot's started/stopped speaking messages should be sent. @@ -907,9 +934,17 @@ class RTVIObserverParams: metrics_enabled: Indicates if metrics messages should be sent. system_logs_enabled: Indicates if system logs should be sent. errors_enabled: [Deprecated] Indicates if errors messages should be sent. + skip_aggregator_types: List of aggregation types to skip sending as tts/output messages. + Note: if using this to avoid sending secure information, be sure to also disable + bot_llm_enabled to avoid leaking through LLM messages. + bot_output_transforms: A list of callables to transform text before just before sending it + to TTS. Each callable takes the aggregated text and its type, and returns the + transformed text. To register, provide a list of tuples of + (aggregation_type | '*', transform_function). audio_level_period_secs: How often audio levels should be sent if enabled. """ + bot_output_enabled: bool = True bot_llm_enabled: bool = True bot_tts_enabled: bool = True bot_speaking_enabled: bool = True @@ -921,6 +956,15 @@ class RTVIObserverParams: metrics_enabled: bool = True system_logs_enabled: bool = False errors_enabled: Optional[bool] = None + skip_aggregator_types: Optional[List[AggregationType | str]] = None + bot_output_transforms: Optional[ + List[ + Tuple[ + AggregationType | str, + Callable[[str, AggregationType | str], Awaitable[str]], + ] + ] + ] = None audio_level_period_secs: float = 0.15 @@ -973,8 +1017,45 @@ class RTVIObserver(BaseObserver): DeprecationWarning, ) + self._aggregation_transforms: List[ + Tuple[AggregationType | str, Callable[[str, AggregationType | str], Awaitable[str]]] + ] = self._params.bot_output_transforms or [] + + def add_bot_output_transformer( + self, + transform_function: Callable[[str, AggregationType | str], Awaitable[str]], + aggregation_type: AggregationType | str = "*", + ): + """Transform text for a specific aggregation type before sending as Bot Output or TTS. + + Args: + transform_function: The function to apply for transformation. This function should take + the text and aggregation type as input and return the transformed text. + Ex.: async def my_transform(text: str, aggregation_type: str) -> str: + aggregation_type: The type of aggregation to transform. This value defaults to "*" to + handle all text before sending to the client. + """ + self._aggregation_transforms.append((aggregation_type, transform_function)) + + def remove_bot_output_transformer( + self, + transform_function: Callable[[str, AggregationType | str], Awaitable[str]], + aggregation_type: AggregationType | str = "*", + ): + """Remove a text transformer for a specific aggregation type. + + Args: + transform_function: The function to remove. + aggregation_type: The type of aggregation to remove the transformer for. + """ + self._aggregation_transforms = [ + (agg_type, func) + for agg_type, func in self._aggregation_transforms + if not (agg_type == aggregation_type and func == transform_function) + ] + async def _logger_sink(self, message): - """Logger sink so we cna send system logs to RTVI clients.""" + """Logger sink so we can send system logs to RTVI clients.""" message = RTVISystemLogMessage(data=RTVITextMessageData(text=message)) await self.send_rtvi_message(message) @@ -1048,12 +1129,15 @@ class RTVIObserver(BaseObserver): await self.send_rtvi_message(RTVIBotTTSStartedMessage()) elif isinstance(frame, TTSStoppedFrame) and self._params.bot_tts_enabled: await self.send_rtvi_message(RTVIBotTTSStoppedMessage()) - elif isinstance(frame, TTSTextFrame) and self._params.bot_tts_enabled: - if isinstance(src, BaseOutputTransport): - message = RTVIBotTTSTextMessage(data=RTVITextMessageData(text=frame.text)) - await self.send_rtvi_message(message) - else: + elif isinstance(frame, AggregatedTextFrame) and ( + self._params.bot_output_enabled or self._params.bot_tts_enabled + ): + if isinstance(frame, TTSTextFrame) and not isinstance(src, BaseOutputTransport): + # This check is to make sure we handle the frame when it has gone + # through the transport and has correct timing. mark_as_seen = False + else: + await self._handle_aggregated_llm_text(frame) elif isinstance(frame, MetricsFrame) and self._params.metrics_enabled: await self._handle_metrics(frame) elif isinstance(frame, RTVIServerMessageFrame): @@ -1084,15 +1168,6 @@ class RTVIObserver(BaseObserver): if mark_as_seen: self._frames_seen.add(frame.id) - async def _push_bot_transcription(self): - """Push accumulated bot transcription as a message.""" - if len(self._bot_transcription) > 0: - message = RTVIBotTranscriptionMessage( - data=RTVITextMessageData(text=self._bot_transcription) - ) - await self.send_rtvi_message(message) - self._bot_transcription = "" - async def _handle_interruptions(self, frame: Frame): """Handle user speaking interruption frames.""" message = None @@ -1115,14 +1190,45 @@ class RTVIObserver(BaseObserver): if message: await self.send_rtvi_message(message) + async def _handle_aggregated_llm_text(self, frame: AggregatedTextFrame): + """Handle aggregated LLM text output frames.""" + # Skip certain aggregator types if configured to do so. + if ( + self._params.skip_aggregator_types + and frame.aggregated_by in self._params.skip_aggregator_types + ): + return + + text = frame.text + type = frame.aggregated_by + for aggregation_type, transform in self._aggregation_transforms: + if aggregation_type == type or aggregation_type == "*": + text = await transform(text, type) + + isTTS = isinstance(frame, TTSTextFrame) + if self._params.bot_output_enabled: + message = RTVIBotOutputMessage( + data=RTVIBotOutputMessageData(text=text, spoken=isTTS, aggregated_by=type) + ) + await self.send_rtvi_message(message) + + if isTTS and self._params.bot_tts_enabled: + tts_message = RTVIBotTTSTextMessage(data=RTVITextMessageData(text=text)) + await self.send_rtvi_message(tts_message) + async def _handle_llm_text_frame(self, frame: LLMTextFrame): """Handle LLM text output frames.""" message = RTVIBotLLMTextMessage(data=RTVITextMessageData(text=frame.text)) await self.send_rtvi_message(message) + # TODO (mrkb): Remove all this logic when we fully deprecate bot-transcription messages. self._bot_transcription += frame.text - if match_endofsentence(self._bot_transcription): - await self._push_bot_transcription() + + if match_endofsentence(self._bot_transcription) and len(self._bot_transcription) > 0: + await self.send_rtvi_message( + RTVIBotTranscriptionMessage(data=RTVITextMessageData(text=self._bot_transcription)) + ) + self._bot_transcription = "" async def _handle_user_transcriptions(self, frame: Frame): """Handle user transcription frames.""" @@ -1248,7 +1354,7 @@ class RTVIProcessor(FrameProcessor): # Default to 0.3.0 which is the last version before actually having a # "client-version". self._client_version = [0, 3, 0] - self._skip_tts: bool = False # Keep in sync with llm_service.py + self._llm_skip_tts: bool = False # Keep in sync with llm_service.py's configuration. self._registered_actions: Dict[str, RTVIAction] = {} self._registered_services: Dict[str, RTVIService] = {} @@ -1441,7 +1547,7 @@ class RTVIProcessor(FrameProcessor): elif isinstance(frame, RTVIActionFrame): await self._action_queue.put(frame) elif isinstance(frame, LLMConfigureOutputFrame): - self._skip_tts = frame.skip_tts + self._llm_skip_tts = frame.skip_tts await self.push_frame(frame, direction) # Other frames else: @@ -1697,9 +1803,9 @@ class RTVIProcessor(FrameProcessor): opts = data.options if data.options is not None else RTVISendTextOptions() if opts.run_immediately: await self.interrupt_bot() - cur_skip_tts = self._skip_tts + cur_llm_skip_tts = self._llm_skip_tts should_skip_tts = not opts.audio_response - toggle_skip_tts = cur_skip_tts != should_skip_tts + toggle_skip_tts = cur_llm_skip_tts != should_skip_tts if toggle_skip_tts: output_frame = LLMConfigureOutputFrame(skip_tts=should_skip_tts) await self.push_frame(output_frame) @@ -1709,7 +1815,7 @@ class RTVIProcessor(FrameProcessor): ) await self.push_frame(text_frame) if toggle_skip_tts: - output_frame = LLMConfigureOutputFrame(skip_tts=cur_skip_tts) + output_frame = LLMConfigureOutputFrame(skip_tts=cur_llm_skip_tts) await self.push_frame(output_frame) async def _handle_update_context(self, data: RTVIAppendToContextData): diff --git a/src/pipecat/processors/frameworks/strands_agents.py b/src/pipecat/processors/frameworks/strands_agents.py index 323a4e921..8022f5387 100644 --- a/src/pipecat/processors/frameworks/strands_agents.py +++ b/src/pipecat/processors/frameworks/strands_agents.py @@ -23,7 +23,7 @@ try: from strands import Agent from strands.multiagent.graph import Graph except ModuleNotFoundError as e: - logger.exception("In order to use Strands Agents, you need to `pip install strands-agents`.") + logger.error("In order to use Strands Agents, you need to `pip install strands-agents`.") raise Exception(f"Missing module: {e}") @@ -143,7 +143,7 @@ class StrandsAgentsProcessor(FrameProcessor): except GeneratorExit: logger.warning(f"{self} generator was closed prematurely") except Exception as e: - logger.exception(f"{self} an unknown error occurred: {e}") + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) finally: if ttfb_tracking: await self.stop_ttfb_metrics() diff --git a/src/pipecat/runner/run.py b/src/pipecat/runner/run.py index ebca467df..083dabc52 100644 --- a/src/pipecat/runner/run.py +++ b/src/pipecat/runner/run.py @@ -302,7 +302,7 @@ def _setup_webrtc_routes( result: StartBotResult = {"sessionId": session_id} if request_data.get("enableDefaultIceServers"): result["iceConfig"] = IceConfig( - iceServers=[IceServer(urls="stun:stun.l.google.com:19302")] + iceServers=[IceServer(urls=["stun:stun.l.google.com:19302"])] ) return result diff --git a/src/pipecat/serializers/plivo.py b/src/pipecat/serializers/plivo.py index 51ae11151..b49acc664 100644 --- a/src/pipecat/serializers/plivo.py +++ b/src/pipecat/serializers/plivo.py @@ -199,7 +199,7 @@ class PlivoFrameSerializer(FrameSerializer): ) except Exception as e: - logger.exception(f"Failed to hang up Plivo call: {e}") + logger.error(f"Failed to hang up Plivo call: {e}") async def deserialize(self, data: str | bytes) -> Frame | None: """Deserializes Plivo WebSocket data to Pipecat frames. diff --git a/src/pipecat/serializers/telnyx.py b/src/pipecat/serializers/telnyx.py index 7603c9bf7..26dca7dac 100644 --- a/src/pipecat/serializers/telnyx.py +++ b/src/pipecat/serializers/telnyx.py @@ -225,7 +225,7 @@ class TelnyxFrameSerializer(FrameSerializer): ) except Exception as e: - logger.exception(f"Failed to hang up Telnyx call: {e}") + logger.error(f"Failed to hang up Telnyx call: {e}") async def deserialize(self, data: str | bytes) -> Frame | None: """Deserializes Telnyx WebSocket data to Pipecat frames. diff --git a/src/pipecat/serializers/twilio.py b/src/pipecat/serializers/twilio.py index d61b29295..240fb2853 100644 --- a/src/pipecat/serializers/twilio.py +++ b/src/pipecat/serializers/twilio.py @@ -236,7 +236,7 @@ class TwilioFrameSerializer(FrameSerializer): ) except Exception as e: - logger.exception(f"Failed to hang up Twilio call: {e}") + logger.error(f"Failed to hang up Twilio call: {e}") async def deserialize(self, data: str | bytes) -> Frame | None: """Deserializes Twilio WebSocket data to Pipecat frames. diff --git a/src/pipecat/services/ai_service.py b/src/pipecat/services/ai_service.py index 759efcf00..70d53822e 100644 --- a/src/pipecat/services/ai_service.py +++ b/src/pipecat/services/ai_service.py @@ -166,6 +166,6 @@ class AIService(FrameProcessor): async for f in generator: if f: if isinstance(f, ErrorFrame): - await self.push_error(f) + await self.push_error_frame(f) else: await self.push_frame(f) diff --git a/src/pipecat/services/anthropic/llm.py b/src/pipecat/services/anthropic/llm.py index 2e3e0272d..a5c67e90e 100644 --- a/src/pipecat/services/anthropic/llm.py +++ b/src/pipecat/services/anthropic/llm.py @@ -458,8 +458,7 @@ class AnthropicLLMService(LLMService): except httpx.TimeoutException: await self._call_event_handler("on_completion_timeout") except Exception as e: - logger.exception(f"{self} exception: {e}") - await self.push_error(ErrorFrame(f"{e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) finally: await self.stop_processing_metrics() await self.push_frame(LLMFullResponseEndFrame()) diff --git a/src/pipecat/services/assemblyai/stt.py b/src/pipecat/services/assemblyai/stt.py index d78a42841..24251319f 100644 --- a/src/pipecat/services/assemblyai/stt.py +++ b/src/pipecat/services/assemblyai/stt.py @@ -206,9 +206,8 @@ class AssemblyAISTTService(STTService): await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"{self} exception: {e}") self._connected = False - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) raise async def _disconnect(self): @@ -233,8 +232,7 @@ class AssemblyAISTTService(STTService): logger.warning("Timed out waiting for termination message from server") except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) if self._receive_task: await self.cancel_task(self._receive_task) @@ -242,8 +240,7 @@ class AssemblyAISTTService(STTService): await self._websocket.close() except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) finally: self._websocket = None @@ -262,13 +259,11 @@ class AssemblyAISTTService(STTService): except websockets.exceptions.ConnectionClosedOK: break except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) break except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) def _parse_message(self, message: Dict[str, Any]) -> BaseMessage: """Parse a raw message into the appropriate message type.""" @@ -297,8 +292,7 @@ class AssemblyAISTTService(STTService): elif isinstance(parsed_message, TerminationMessage): await self._handle_termination(parsed_message) except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) async def _handle_termination(self, message: TerminationMessage): """Handle termination message.""" diff --git a/src/pipecat/services/asyncai/tts.py b/src/pipecat/services/asyncai/tts.py index f916d9bdd..458974ad4 100644 --- a/src/pipecat/services/asyncai/tts.py +++ b/src/pipecat/services/asyncai/tts.py @@ -56,6 +56,17 @@ def language_to_async_language(language: Language) -> Optional[str]: Language.ES: "es", Language.DE: "de", Language.IT: "it", + Language.PT: "pt", + Language.NL: "nl", + Language.AR: "ar", + Language.RU: "ru", + Language.RO: "ro", + Language.JA: "ja", + Language.HE: "he", + Language.HY: "hy", + Language.TR: "tr", + Language.HI: "hi", + Language.ZH: "zh", } return resolve_language(language, LANGUAGE_MAP, use_base_code=True) @@ -74,7 +85,7 @@ class AsyncAITTSService(InterruptibleTTSService): language: Language to use for synthesis. """ - language: Optional[Language] = Language.EN + language: Optional[Language] = None def __init__( self, @@ -83,7 +94,7 @@ class AsyncAITTSService(InterruptibleTTSService): voice_id: str, version: str = "v1", url: str = "wss://api.async.ai/text_to_speech/websocket/ws", - model: str = "asyncflow_v2.0", + model: str = "asyncflow_multilingual_v1.0", sample_rate: Optional[int] = None, encoding: str = "pcm_s16le", container: str = "raw", @@ -99,7 +110,7 @@ class AsyncAITTSService(InterruptibleTTSService): https://docs.async.ai/list-voices-16699698e0 version: Async API version. url: WebSocket URL for Async TTS API. - model: TTS model to use (e.g., "asyncflow_v2.0"). + model: TTS model to use (e.g., "asyncflow_multilingual_v1.0"). sample_rate: Audio sample rate. encoding: Audio encoding format. container: Audio container format. @@ -128,7 +139,7 @@ class AsyncAITTSService(InterruptibleTTSService): }, "language": self.language_to_service_language(params.language) if params.language - else "en", + else None, } self.set_model_name(model) @@ -228,8 +239,7 @@ class AsyncAITTSService(InterruptibleTTSService): await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) self._websocket = None await self._call_event_handler("on_connection_error", f"{e}") @@ -241,8 +251,7 @@ class AsyncAITTSService(InterruptibleTTSService): logger.debug("Disconnecting from Async") await self._websocket.close() except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) finally: self._websocket = None self._started = False @@ -287,12 +296,11 @@ class AsyncAITTSService(InterruptibleTTSService): ) await self.push_frame(frame) elif msg.get("error_code"): - logger.error(f"{self} error: {msg}") await self.push_frame(TTSStoppedFrame()) await self.stop_all_metrics() - await self.push_error(ErrorFrame(error=f"{self} error: {msg['message']}")) + await self.push_error(error_msg=f"Error: {msg['message']}") else: - logger.error(f"{self} error, unknown message type: {msg}") + await self.push_error(error_msg=f"Unknown message type: {msg}") async def _keepalive_task_handler(self): """Send periodic keepalive messages to maintain WebSocket connection.""" @@ -335,16 +343,14 @@ class AsyncAITTSService(InterruptibleTTSService): await self._get_websocket().send(msg) await self.start_tts_usage_metrics(text) except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") yield TTSStoppedFrame() await self._disconnect() await self._connect() return yield None except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") class AsyncAIHttpTTSService(TTSService): @@ -362,7 +368,7 @@ class AsyncAIHttpTTSService(TTSService): language: Language to use for synthesis. """ - language: Optional[Language] = Language.EN + language: Optional[Language] = None def __init__( self, @@ -370,7 +376,7 @@ class AsyncAIHttpTTSService(TTSService): api_key: str, voice_id: str, aiohttp_session: aiohttp.ClientSession, - model: str = "asyncflow_v2.0", + model: str = "asyncflow_multilingual_v1.0", url: str = "https://api.async.ai", version: str = "v1", sample_rate: Optional[int] = None, @@ -385,7 +391,7 @@ class AsyncAIHttpTTSService(TTSService): api_key: Async API key. voice_id: ID of the voice to use for synthesis. aiohttp_session: An aiohttp session for making HTTP requests. - model: TTS model to use (e.g., "asyncflow_v2.0"). + model: TTS model to use (e.g., "asyncflow_multilingual_v1.0"). url: Base URL for Async API. version: API version string for Async API. sample_rate: Audio sample rate. @@ -409,7 +415,7 @@ class AsyncAIHttpTTSService(TTSService): }, "language": self.language_to_service_language(params.language) if params.language - else "en", + else None, } self.set_voice(voice_id) self.set_model_name(model) @@ -477,8 +483,7 @@ class AsyncAIHttpTTSService(TTSService): async with self._session.post(url, json=payload, headers=headers) as response: if response.status != 200: error_text = await response.text() - logger.error(f"Async API error: {error_text}") - await self.push_error(ErrorFrame(error=f"Async API error: {error_text}")) + await self.push_error(error_msg=f"Async API error: {error_text}") raise Exception(f"Async API returned status {response.status}: {error_text}") audio_data = await response.read() @@ -494,8 +499,7 @@ class AsyncAIHttpTTSService(TTSService): yield frame except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) finally: await self.stop_ttfb_metrics() yield TTSStoppedFrame() diff --git a/src/pipecat/services/aws/__init__.py b/src/pipecat/services/aws/__init__.py index 3cdd4cc5a..88725f965 100644 --- a/src/pipecat/services/aws/__init__.py +++ b/src/pipecat/services/aws/__init__.py @@ -8,8 +8,10 @@ import sys from pipecat.services import DeprecatedModuleProxy +from .agent_core import * from .llm import * from .nova_sonic import * +from .sagemaker import * from .stt import * from .tts import * diff --git a/src/pipecat/services/aws/agent_core.py b/src/pipecat/services/aws/agent_core.py new file mode 100644 index 000000000..be4806221 --- /dev/null +++ b/src/pipecat/services/aws/agent_core.py @@ -0,0 +1,258 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""AWS AgentCore Processor Module. + +This module defines the AWSAgentCoreProcessor, which invokes agents hosted on +Amazon Bedrock AgentCore Runtime and streams their responses as LLMTextFrames. +""" + +import asyncio +import json +import os +from typing import Callable, Optional + +import aioboto3 +from loguru import logger + +from pipecat.frames.frames import ( + Frame, + LLMContextFrame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMTextFrame, +) +from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage +from pipecat.processors.aggregators.openai_llm_context import ( + OpenAILLMContext, + OpenAILLMContextFrame, +) +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor + + +def default_context_to_payload_transformer( + context: LLMContext | OpenAILLMContext, +) -> Optional[str]: + """Default transformer to create AgentCore payload from LLM context. + + Extracts the latest user or system message text and wraps it in {"prompt": ""}. + + Args: + context: The LLM context containing conversation messages. + + Returns: + A JSON string payload for AgentCore, or None if no valid message found. + """ + messages = context.messages + + if not messages: + return None + + last_message = messages[-1] + if isinstance(last_message, LLMSpecificMessage) or last_message.get("role") not in ( + "user", + "system", + ): + return None + + content = last_message.get("content") + if not content: + return None + + if isinstance(content, str): + prompt = content + elif isinstance(content, list): + prompt = " ".join([part.get("text", "") for part in content]) + else: + return None + + return json.dumps({"prompt": prompt}) + + +def default_response_to_output_transformer(response_line: str) -> Optional[str]: + """Default transformer to extract output text from AgentCore response. + + Expects responses with {"response": ""} format. + + Args: + response_line: The raw response line from AgentCore (without "data: " prefix). + + Returns: + The extracted output text, or None if no text found. + """ + response_json = json.loads(response_line) + return response_json.get("response") + + +class AWSAgentCoreProcessor(FrameProcessor): + """Processor that runs an Amazon Bedrock AgentCore agent. + + Input: + - LLMContextFrame: Supplies a context used to invoke the agent. + + Output: + - LLMTextFrame: The agent's text response(s). + A single agent invocation may result in multiple text frames. + + This processor transforms the input context to a payload for the AgentCore + agent, and transforms the agent's response(s) into output text frame(s). Both + mappings are configurable via transformers. Below is the default behavior. + + Input transformer (context_to_payload_transformer): + - Grabs the latest user or system message (if it's the latest message) + - Extracts its text content + - Constructs a payload that looks like {"prompt": ""} + + Output transformer (response_to_output_transformer): + - Expects responses that look like {"response": ""} + - Extracts the text for use in the LLMTextFrame(s) + """ + + def __init__( + self, + agentArn: str, + aws_access_key: Optional[str] = None, + aws_secret_key: Optional[str] = None, + aws_session_token: Optional[str] = None, + aws_region: Optional[str] = None, + context_to_payload_transformer: Optional[ + Callable[[LLMContext | OpenAILLMContext], Optional[str]] + ] = None, + response_to_output_transformer: Optional[Callable[[str], Optional[str]]] = None, + **kwargs, + ): + """Initialize the AWS AgentCore processor. + + Args: + agentArn: The Amazon Web Services Resource Name (ARN) of the agent. + aws_access_key: AWS access key ID. If None, uses default credentials. + aws_secret_key: AWS secret access key. If None, uses default credentials. + aws_session_token: AWS session token for temporary credentials. + aws_region: AWS region. + context_to_payload_transformer: Optional callable to transform + LLMContext into AgentCore payload string. If None, uses + default_context_to_payload_transformer. + response_to_output_transformer: Optional callable to extract output text + from AgentCore response. If None, uses + default_response_to_output_transformer. + **kwargs: Additional arguments passed to parent FrameProcessor. + """ + super().__init__(**kwargs) + + self._agentArn = agentArn + self._aws_session = aioboto3.Session() + + # Store AWS session parameters for creating client in async context + self._aws_params = { + "aws_access_key_id": aws_access_key or os.getenv("AWS_ACCESS_KEY_ID"), + "aws_secret_access_key": aws_secret_key or os.getenv("AWS_SECRET_ACCESS_KEY"), + "aws_session_token": aws_session_token or os.getenv("AWS_SESSION_TOKEN"), + "region_name": aws_region or os.getenv("AWS_REGION", "us-east-1"), + } + + # Set transformers with defaults + self._context_to_payload_transformer = ( + context_to_payload_transformer or default_context_to_payload_transformer + ) + self._response_to_output_transformer = ( + response_to_output_transformer or default_response_to_output_transformer + ) + + # State for managing output response bookends + self._output_response_open = False + self._last_text_frame_time: Optional[float] = None + self._close_task: Optional[asyncio.Task] = None + self._output_response_timeout = 1.0 # seconds + + async def _close_output_response_after_timeout(self): + """Close the output response after timeout if no new text frames arrive.""" + await asyncio.sleep(self._output_response_timeout) + if self._output_response_open: + self._output_response_open = False + await self.push_frame(LLMFullResponseEndFrame()) + + async def _push_text_frame(self, text: str): + """Push a text frame, managing output response bookends.""" + # Cancel any pending close task + if self._close_task and not self._close_task.done(): + await self.cancel_task(self._close_task) + + # Open output response if needed + if not self._output_response_open: + await self.push_frame(LLMFullResponseStartFrame()) + self._output_response_open = True + + # Push the text frame + await self.push_frame(LLMTextFrame(text)) + self._last_text_frame_time = asyncio.get_event_loop().time() + + # Schedule closing the output response after timeout + self._close_task = self.create_task(self._close_output_response_after_timeout()) + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and handle LLM message frames. + + Args: + frame: The incoming frame to process. + direction: The direction of frame flow in the pipeline. + """ + await super().process_frame(frame, direction) + if isinstance(frame, (LLMContextFrame, OpenAILLMContextFrame)): + # Create payload to invoke AgentCore agent + payload = self._context_to_payload_transformer(frame.context) + + if not payload: + return + + async with self._aws_session.client("bedrock-agentcore", **self._aws_params) as client: + # Invoke the AgentCore agent + response = await client.invoke_agent_runtime( + agentRuntimeArn=self._agentArn, payload=payload.encode() + ) + + # Determine if this is a streamed multi-part response, which + # will affect our parsing + is_multi_part_response = "text/event-stream" in response.get("contentType", "") + + # Handle each response part (there may be one, for single + # responses, or multiple, for streamed multi-part responses) + async for part in response.get("response", []): + part_string = part.decode("utf-8") + + # In streamed multi-part responses, each part might have + # one or more lines, each of which starts with "data: ". + # Treat each line as a response. + if is_multi_part_response: + for line in part_string.split("\n"): + # Get response text from this line + if not line: + continue + if not line.startswith("data: "): + logger.warning(f"Expected line to start with 'data: ', got: {line}") + continue + line = line[6:] # omit "data: " + + # Transform response line to output text + text = self._response_to_output_transformer(line) + if text: + await self._push_text_frame(text) + + # In single-part responses, the whole part is one response + # and there's no "data: " prefix + else: + # Transform response part string to output text + text = self._response_to_output_transformer(part_string) + if text: + await self._push_text_frame(text) + + # Final close if output response is still open after all parts processed + if self._output_response_open: + if self._close_task and not self._close_task.done(): + await self.cancel_task(self._close_task) + self._output_response_open = False + await self.push_frame(LLMFullResponseEndFrame()) + else: + await self.push_frame(frame, direction) diff --git a/src/pipecat/services/aws/llm.py b/src/pipecat/services/aws/llm.py index ccbac43b7..e5488ed34 100644 --- a/src/pipecat/services/aws/llm.py +++ b/src/pipecat/services/aws/llm.py @@ -734,7 +734,7 @@ class AWSBedrockLLMService(LLMService): aws_access_key: Optional[str] = None, aws_secret_key: Optional[str] = None, aws_session_token: Optional[str] = None, - aws_region: str = "us-east-1", + aws_region: Optional[str] = None, params: Optional[InputParams] = None, client_config: Optional[Config] = None, retry_timeout_secs: Optional[float] = 5.0, @@ -1136,7 +1136,7 @@ class AWSBedrockLLMService(LLMService): except (ReadTimeoutError, asyncio.TimeoutError): await self._call_event_handler("on_completion_timeout") except Exception as e: - logger.exception(f"{self} exception: {e}") + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) finally: await self.stop_processing_metrics() await self.push_frame(LLMFullResponseEndFrame()) diff --git a/src/pipecat/services/aws/nova_sonic/llm.py b/src/pipecat/services/aws/nova_sonic/llm.py index 2572b03cb..d08d2603f 100644 --- a/src/pipecat/services/aws/nova_sonic/llm.py +++ b/src/pipecat/services/aws/nova_sonic/llm.py @@ -27,6 +27,7 @@ from pydantic import BaseModel, Field from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.services.aws_nova_sonic_adapter import AWSNovaSonicLLMAdapter, Role from pipecat.frames.frames import ( + AggregationType, BotStoppedSpeakingFrame, CancelFrame, EndFrame, @@ -452,7 +453,7 @@ class AWSNovaSonicLLMService(LLMService): self._ready_to_send_context = True await self._finish_connecting_if_context_available() except Exception as e: - logger.error(f"{self} initialization error: {e}") + await self.push_error(error_msg=f"Initialization error: {e}", exception=e) await self._disconnect() async def _process_completed_function_calls(self, send_new_results: bool): @@ -576,7 +577,7 @@ class AWSNovaSonicLLMService(LLMService): logger.info("Finished disconnecting") except Exception as e: - logger.error(f"{self} error disconnecting: {e}") + await self.push_error(error_msg=f"Error disconnecting: {e}", exception=e) def _create_client(self) -> BedrockRuntimeClient: config = Config( @@ -884,7 +885,7 @@ class AWSNovaSonicLLMService(LLMService): # Errors are kind of expected while disconnecting, so just # ignore them and do nothing return - logger.error(f"{self} error processing responses: {e}") + await self.push_error(error_msg=f"Error processing responses: {e}", exception=e) if self._wants_connection: await self.reset_conversation() @@ -1027,7 +1028,7 @@ class AWSNovaSonicLLMService(LLMService): logger.debug(f"Assistant response text added: {text}") # Report the text of the assistant response. - frame = TTSTextFrame(text) + frame = TTSTextFrame(text, aggregated_by=AggregationType.SENTENCE) frame.includes_inter_frame_spaces = True await self.push_frame(frame) @@ -1062,7 +1063,9 @@ class AWSNovaSonicLLMService(LLMService): # TTSTextFrame would be ignored otherwise (the interruption frame # would have cleared the assistant aggregator state). await self.push_frame(LLMFullResponseStartFrame()) - frame = TTSTextFrame(self._assistant_text_buffer) + frame = TTSTextFrame( + self._assistant_text_buffer, aggregated_by=AggregationType.SENTENCE + ) frame.includes_inter_frame_spaces = True await self.push_frame(frame) self._may_need_repush_assistant_text = False diff --git a/src/pipecat/services/aws/sagemaker/__init__.py b/src/pipecat/services/aws/sagemaker/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/pipecat/services/aws/sagemaker/bidi_client.py b/src/pipecat/services/aws/sagemaker/bidi_client.py new file mode 100644 index 000000000..5e02af03d --- /dev/null +++ b/src/pipecat/services/aws/sagemaker/bidi_client.py @@ -0,0 +1,283 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""AWS SageMaker bidirectional streaming client. + +This module provides a client for streaming bidirectional communication with +SageMaker endpoints using the HTTP/2 protocol. Supports sending audio, text, +and JSON data to SageMaker model endpoints and receiving streaming responses. +""" + +import os +from typing import Optional + +from loguru import logger + +try: + from aws_sdk_sagemaker_runtime_http2.client import SageMakerRuntimeHTTP2Client + from aws_sdk_sagemaker_runtime_http2.config import Config, HTTPAuthSchemeResolver + from aws_sdk_sagemaker_runtime_http2.models import ( + InvokeEndpointWithBidirectionalStreamInput, + RequestPayloadPart, + RequestStreamEventPayloadPart, + ResponseStreamEvent, + ) + from smithy_aws_core.auth.sigv4 import SigV4AuthScheme + from smithy_aws_core.identity import EnvironmentCredentialsResolver + from smithy_core.aio.eventstream import DuplexEventStream +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error( + "In order to use SageMaker BiDi client, you need to `pip install pipecat-ai[sagemaker]`." + ) + raise Exception(f"Missing module: {e}") + + +class SageMakerBidiClient: + """Client for bidirectional streaming with AWS SageMaker endpoints. + + Handles low-level HTTP/2 bidirectional streaming protocol for communicating + with SageMaker model endpoints. Provides methods for sending various data + types (audio, text, JSON) and receiving streaming responses. + + This client uses AWS SigV4 authentication and supports credential resolution + from environment variables, AWS CLI configuration, and instance metadata. + + Example:: + + client = SageMakerBidiClient( + endpoint_name="my-deepgram-endpoint", + region="us-east-2", + model_invocation_path="v1/listen", + model_query_string="model=nova-3&language=en" + ) + await client.start_session() + await client.send_audio_chunk(audio_bytes) + response = await client.receive_response() + await client.close_session() + """ + + def __init__( + self, + endpoint_name: str, + region: str, + model_invocation_path: str = "", + model_query_string: str = "", + ): + """Initialize the SageMaker BiDi client. + + Args: + endpoint_name: Name of the SageMaker endpoint to connect to. + region: AWS region where the endpoint is deployed. + model_invocation_path: API path for the model invocation (e.g., "v1/listen"). + model_query_string: Query string parameters for the model (e.g., "model=nova-3"). + """ + self.endpoint_name = endpoint_name + self.region = region + self.model_invocation_path = model_invocation_path + self.model_query_string = model_query_string + self.bidi_endpoint = f"https://runtime.sagemaker.{region}.amazonaws.com:8443" + self._client: Optional[SageMakerRuntimeHTTP2Client] = None + self._stream: Optional[ + DuplexEventStream[RequestStreamEventPayloadPart, ResponseStreamEvent, any] + ] = None + self._output_stream = None + self._is_active = False + + def _initialize_client(self): + """Initialize the SageMaker Runtime HTTP2 client with AWS credentials. + + Creates and configures the SageMaker Runtime HTTP2 client with SigV4 + authentication. Attempts to resolve AWS credentials from environment + variables, AWS CLI configuration, or instance metadata. + """ + logger.debug(f"Initializing SageMaker BiDi client for region: {self.region}") + logger.debug(f"Using endpoint URI: {self.bidi_endpoint}") + + # Check for AWS credentials + has_env_creds = bool(os.getenv("AWS_ACCESS_KEY_ID") and os.getenv("AWS_SECRET_ACCESS_KEY")) + + if not has_env_creds: + logger.warning( + "AWS credentials not found in environment variables. " + "Attempting to use EnvironmentCredentialsResolver which will check " + "AWS CLI configuration and instance metadata." + ) + + config = Config( + endpoint_uri=self.bidi_endpoint, + region=self.region, + aws_credentials_identity_resolver=EnvironmentCredentialsResolver(), + auth_scheme_resolver=HTTPAuthSchemeResolver(), + auth_schemes={"aws.auth#sigv4": SigV4AuthScheme(service="sagemaker")}, + ) + self._client = SageMakerRuntimeHTTP2Client(config=config) + + async def start_session(self): + """Start a bidirectional streaming session with the SageMaker endpoint. + + Initializes the client if needed, creates the bidirectional stream, and + establishes the connection to the SageMaker endpoint. Must be called + before sending or receiving data. + + Returns: + The output stream for receiving responses. + + Raises: + RuntimeError: If client initialization or connection fails. + """ + if not self._client: + self._initialize_client() + + logger.debug(f"Starting BiDi session with endpoint: {self.endpoint_name}") + logger.debug(f"Model invocation path: {self.model_invocation_path}") + logger.debug(f"Model query string: {self.model_query_string}") + + # Create the bidirectional stream + stream_input = InvokeEndpointWithBidirectionalStreamInput( + endpoint_name=self.endpoint_name, + model_invocation_path=self.model_invocation_path, + model_query_string=self.model_query_string, + ) + + try: + self._stream = await self._client.invoke_endpoint_with_bidirectional_stream( + stream_input + ) + self._is_active = True + + # Get output stream + output = await self._stream.await_output() + self._output_stream = output[1] + + logger.debug("BiDi session started successfully") + return self._output_stream + + except Exception as e: + logger.error(f"Failed to start BiDi session: {e}") + self._is_active = False + raise RuntimeError(f"Failed to start SageMaker BiDi session: {e}") + + async def send_data(self, data_bytes: bytes, data_type: Optional[str] = None): + """Send a chunk of data to the stream. + + Generic method for sending any type of data to the SageMaker endpoint. + Use the convenience methods (send_audio_chunk, send_text, send_json) + for common data types. + + Args: + data_bytes: Raw bytes to send. + data_type: Optional data type header. Common values are "BINARY" for + audio/binary data and "UTF8" for text/JSON data. + + Raises: + RuntimeError: If session is not active or send fails. + """ + if not self._is_active or not self._stream: + raise RuntimeError("BiDi session not active") + + try: + payload = RequestPayloadPart(bytes_=data_bytes, data_type=data_type) + event = RequestStreamEventPayloadPart(value=payload) + await self._stream.input_stream.send(event) + except Exception as e: + logger.error(f"Failed to send data: {e}") + raise + + async def send_audio_chunk(self, audio_bytes: bytes): + """Send a chunk of audio data to the stream. + + Convenience method for sending audio data. Automatically sets the data + type to "BINARY". + + Args: + audio_bytes: Raw audio bytes to send (e.g., PCM audio data). + + Raises: + RuntimeError: If session is not active or send fails. + """ + await self.send_data(audio_bytes, data_type="BINARY") + + async def send_text(self, text: str): + """Send text data to the stream. + + Convenience method for sending text data. Automatically encodes the text + as UTF-8 and sets the data type to "UTF8". + + Args: + text: Text string to send. + + Raises: + RuntimeError: If session is not active or send fails. + """ + await self.send_data(text.encode("utf-8"), data_type="UTF8") + + async def send_json(self, data: dict): + """Send JSON data to the stream. + + Convenience method for sending JSON-encoded messages. Useful for control + messages like KeepAlive or CloseStream. Automatically serializes the + dictionary to JSON, encodes as UTF-8, and sets the data type to "UTF8". + + Args: + data: Dictionary to send as JSON (e.g., {"type": "KeepAlive"}). + + Raises: + RuntimeError: If session is not active or send fails. + """ + import json + + await self.send_data(json.dumps(data).encode("utf-8"), data_type="UTF8") + + async def receive_response(self) -> Optional[ResponseStreamEvent]: + """Receive a response from the stream. + + Blocks until a response is available from the SageMaker endpoint. Returns + None when the stream is closed. + + Returns: + The response event containing payload data, or None if stream is closed. + + Raises: + RuntimeError: If session is not active. + """ + if not self._is_active or not self._output_stream: + raise RuntimeError("BiDi session not active") + + try: + result = await self._output_stream.receive() + return result + except Exception as e: + logger.error(f"Failed to receive response: {e}") + raise + + async def close_session(self): + """Close the bidirectional streaming session. + + Gracefully closes the input stream and marks the session as inactive. + Safe to call multiple times. + """ + if not self._is_active: + return + + logger.debug("Closing BiDi session...") + self._is_active = False + + try: + if self._stream: + await self._stream.input_stream.close() + logger.debug("BiDi session closed successfully") + except Exception as e: + logger.warning(f"Error closing BiDi session: {e}") + + @property + def is_active(self) -> bool: + """Check if the session is currently active. + + Returns: + True if session is active, False otherwise. + """ + return self._is_active diff --git a/src/pipecat/services/aws/stt.py b/src/pipecat/services/aws/stt.py index 8d89d3198..0878db315 100644 --- a/src/pipecat/services/aws/stt.py +++ b/src/pipecat/services/aws/stt.py @@ -58,7 +58,7 @@ class AWSTranscribeSTTService(STTService): api_key: Optional[str] = None, aws_access_key_id: Optional[str] = None, aws_session_token: Optional[str] = None, - region: Optional[str] = "us-east-1", + region: Optional[str] = None, sample_rate: int = 16000, language: Language = Language.EN, **kwargs, @@ -69,7 +69,7 @@ class AWSTranscribeSTTService(STTService): api_key: AWS secret access key. If None, uses AWS_SECRET_ACCESS_KEY environment variable. aws_access_key_id: AWS access key ID. If None, uses AWS_ACCESS_KEY_ID environment variable. aws_session_token: AWS session token for temporary credentials. If None, uses AWS_SESSION_TOKEN environment variable. - region: AWS region for the service. Defaults to "us-east-1". + region: AWS region for the service. sample_rate: Audio sample rate in Hz. Must be 8000 or 16000. Defaults to 16000. language: Language for transcription. Defaults to English. **kwargs: Additional arguments passed to parent STTService class. @@ -140,8 +140,7 @@ class AWSTranscribeSTTService(STTService): return logger.warning("WebSocket connection not established after connect") except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) retry_count += 1 if retry_count < max_retries: await asyncio.sleep(1) # Wait before retrying @@ -182,8 +181,7 @@ class AWSTranscribeSTTService(STTService): try: await self._connect() except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") return # Format the audio data according to AWS event stream format @@ -200,13 +198,11 @@ class AWSTranscribeSTTService(STTService): await self._disconnect() # Don't yield error here - we'll retry on next frame except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") await self._disconnect() except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") await self._disconnect() async def _connect(self): @@ -289,8 +285,7 @@ class AWSTranscribeSTTService(STTService): await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) await self._disconnect() raise @@ -310,8 +305,7 @@ class AWSTranscribeSTTService(STTService): await self._ws_client.send(json.dumps(end_stream)) await self._ws_client.close() except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) finally: self._ws_client = None await self._call_event_handler("on_disconnected") @@ -529,15 +523,15 @@ class AWSTranscribeSTTService(STTService): ) elif headers.get(":message-type") == "exception": error_msg = payload.get("Message", "Unknown error") - logger.error(f"{self} Exception from AWS: {error_msg}") - await self.push_frame(ErrorFrame(f"AWS Transcribe error: {error_msg}")) + await self.push_error(error_msg=f"AWS Transcribe error: {error_msg}") else: logger.debug(f"{self} Other message type received: {headers}") logger.debug(f"{self} Payload: {payload}") except websockets.exceptions.ConnectionClosed as e: - logger.error(f"{self} WebSocket connection closed in receive loop: {e}") + await self.push_error( + error_msg=f"WebSocket connection closed in receive loop", exception=e + ) break except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) break diff --git a/src/pipecat/services/aws/tts.py b/src/pipecat/services/aws/tts.py index f22c42399..7e319abea 100644 --- a/src/pipecat/services/aws/tts.py +++ b/src/pipecat/services/aws/tts.py @@ -312,7 +312,6 @@ class AWSPollyTTSService(TTSService): yield TTSStoppedFrame() except (BotoCoreError, ClientError) as error: - logger.exception(f"{self} error generating TTS: {error}") error_message = f"AWS Polly TTS error: {str(error)}" yield ErrorFrame(error=error_message) diff --git a/src/pipecat/services/azure/image.py b/src/pipecat/services/azure/image.py index f07b86f83..6b0b140e0 100644 --- a/src/pipecat/services/azure/image.py +++ b/src/pipecat/services/azure/image.py @@ -91,7 +91,6 @@ class AzureImageGenServiceREST(ImageGenService): while status != "succeeded": attempts_left -= 1 if attempts_left == 0: - logger.error(f"{self} error: image generation timed out") yield ErrorFrame("Image generation timed out") return @@ -104,7 +103,6 @@ class AzureImageGenServiceREST(ImageGenService): image_url = json_response["result"]["data"][0]["url"] if json_response else None if not image_url: - logger.error(f"{self} error: image generation failed") yield ErrorFrame("Image generation failed") return diff --git a/src/pipecat/services/azure/realtime/llm.py b/src/pipecat/services/azure/realtime/llm.py index 66ba95eea..b06a05c64 100644 --- a/src/pipecat/services/azure/realtime/llm.py +++ b/src/pipecat/services/azure/realtime/llm.py @@ -61,5 +61,5 @@ class AzureRealtimeLLMService(OpenAIRealtimeLLMService): ) self._receive_task = self.create_task(self._receive_task_handler()) except Exception as e: - logger.error(f"{self} initialization error: {e}") + await self.push_error(error_msg=f"initialization error: {e}", exception=e) self._websocket = None diff --git a/src/pipecat/services/azure/stt.py b/src/pipecat/services/azure/stt.py index 85a0508a0..d8186b354 100644 --- a/src/pipecat/services/azure/stt.py +++ b/src/pipecat/services/azure/stt.py @@ -121,8 +121,7 @@ class AzureSTTService(STTService): self._audio_stream.write(audio) yield None except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") async def start(self, frame: StartFrame): """Start the speech recognition service. @@ -151,8 +150,9 @@ class AzureSTTService(STTService): self._speech_recognizer.recognized.connect(self._on_handle_recognized) self._speech_recognizer.start_continuous_recognition_async() except Exception as e: - logger.error(f"{self} exception during initialization: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error( + error_msg=f"Uncaught exception during initialization: {e}", exception=e + ) async def stop(self, frame: EndFrame): """Stop the speech recognition service. diff --git a/src/pipecat/services/azure/tts.py b/src/pipecat/services/azure/tts.py index a1040f312..154930fac 100644 --- a/src/pipecat/services/azure/tts.py +++ b/src/pipecat/services/azure/tts.py @@ -327,7 +327,6 @@ class AzureTTSService(AzureBaseTTSService): try: if self._speech_synthesizer is None: error_msg = "Speech synthesizer not initialized." - logger.error(error_msg) yield ErrorFrame(error=error_msg) return @@ -355,15 +354,13 @@ class AzureTTSService(AzureBaseTTSService): yield TTSStoppedFrame() except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") yield TTSStoppedFrame() # Could add reconnection logic here if needed return except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") class AzureHttpTTSService(AzureBaseTTSService): @@ -440,5 +437,6 @@ class AzureHttpTTSService(AzureBaseTTSService): cancellation_details = result.cancellation_details logger.warning(f"Speech synthesis canceled: {cancellation_details.reason}") if cancellation_details.reason == CancellationReason.Error: - logger.error(f"{self} error: {cancellation_details.error_details}") - yield ErrorFrame(error=f"{self} error: {cancellation_details.error_details}") + yield ErrorFrame( + error=f"Unknown error occurred: {cancellation_details.error_details}" + ) diff --git a/src/pipecat/services/cartesia/stt.py b/src/pipecat/services/cartesia/stt.py index a2ae9432f..3099708e4 100644 --- a/src/pipecat/services/cartesia/stt.py +++ b/src/pipecat/services/cartesia/stt.py @@ -10,7 +10,6 @@ This module provides a WebSocket-based STT service that integrates with the Cartesia Live transcription API for real-time speech recognition. """ -import asyncio import json import urllib.parse from typing import AsyncGenerator, Optional @@ -20,7 +19,6 @@ from loguru import logger from pipecat.frames.frames import ( CancelFrame, EndFrame, - ErrorFrame, Frame, InterimTranscriptionFrame, StartFrame, @@ -160,20 +158,16 @@ class CartesiaSTTService(WebsocketSTTService): sample_rate=sample_rate, ) - merged_options = default_options + merged_options = default_options.to_dict() if live_options: - merged_options_dict = default_options.to_dict() - merged_options_dict.update(live_options.to_dict()) - merged_options = CartesiaLiveOptions( - **{ - k: v - for k, v in merged_options_dict.items() - if not isinstance(v, str) or v != "None" - } - ) + merged_options.update(live_options.to_dict()) + # Filter out "None" string values + merged_options = { + k: v for k, v in merged_options.items() if not isinstance(v, str) or v != "None" + } self._settings = merged_options - self.set_model_name(merged_options.model) + self.set_model_name(merged_options["model"]) self._api_key = api_key self._base_url = base_url or "api.cartesia.ai" self._receive_task = None @@ -254,7 +248,7 @@ class CartesiaSTTService(WebsocketSTTService): await self._connect_websocket() if self._websocket and not self._receive_task: - self._receive_task = asyncio.create_task(self._receive_task_handler(self._report_error)) + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) async def _disconnect(self): if self._receive_task: @@ -269,15 +263,14 @@ class CartesiaSTTService(WebsocketSTTService): return logger.debug("Connecting to Cartesia STT") - params = self._settings.to_dict() + params = self._settings ws_url = f"wss://{self._base_url}/stt/websocket?{urllib.parse.urlencode(params)}" headers = {"Cartesia-Version": "2025-04-16", "X-API-Key": self._api_key} self._websocket = await websocket_connect(ws_url, additional_headers=headers) await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) async def _disconnect_websocket(self): try: @@ -285,8 +278,7 @@ class CartesiaSTTService(WebsocketSTTService): logger.debug("Disconnecting from Cartesia STT") await self._websocket.close() except Exception as e: - logger.error(f"{self} error closing websocket: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Error closing websocket: {e}", exception=e) finally: self._websocket = None await self._call_event_handler("on_disconnected") @@ -297,12 +289,15 @@ class CartesiaSTTService(WebsocketSTTService): raise Exception("Websocket not connected") async def _process_messages(self): + """Process incoming WebSocket messages.""" async for message in self._get_websocket(): try: data = json.loads(message) await self._process_response(data) except json.JSONDecodeError: logger.warning(f"Received non-JSON message: {message}") + except Exception as e: + logger.error(f"Error processing message: {e}") async def _receive_messages(self): while True: @@ -319,8 +314,7 @@ class CartesiaSTTService(WebsocketSTTService): elif data["type"] == "error": error_msg = data.get("message", "Unknown error") - logger.error(f"Cartesia error: {error_msg}") - await self.push_error(ErrorFrame(error=error_msg)) + await self.push_error(error_msg=error_msg) @traced_stt async def _handle_transcription( @@ -352,6 +346,7 @@ class CartesiaSTTService(WebsocketSTTService): self._user_id, time_now_iso8601(), language, + result=data, ) ) await self._handle_transcription(transcript, is_final, language) @@ -364,5 +359,6 @@ class CartesiaSTTService(WebsocketSTTService): self._user_id, time_now_iso8601(), language, + result=data, ) ) diff --git a/src/pipecat/services/cartesia/tts.py b/src/pipecat/services/cartesia/tts.py index f8881200c..9a47d9237 100644 --- a/src/pipecat/services/cartesia/tts.py +++ b/src/pipecat/services/cartesia/tts.py @@ -10,7 +10,8 @@ import base64 import json import uuid import warnings -from typing import AsyncGenerator, List, Literal, Optional, Union +from enum import Enum +from typing import AsyncGenerator, List, Literal, Optional from loguru import logger from pydantic import BaseModel, Field @@ -125,6 +126,72 @@ def language_to_cartesia_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +class CartesiaEmotion(str, Enum): + """Predefined Emotions supported by Cartesia.""" + + # Primary emotions supported by Cartesia + NEUTRAL = "neutral" + ANGRY = "angry" + EXCITED = "excited" + CONTENT = "content" + SAD = "sad" + SCARED = "scared" + # Additional emotions supported by Cartesia + HAPPY = "happy" + ENTHUSIASTIC = "enthusiastic" + ELATED = "elated" + EUPHORIC = "euphoric" + TRIUMPHANT = "triumphant" + AMAZED = "amazed" + SURPRISED = "surprised" + FLIRTATIOUS = "flirtatious" + JOKING_COMEDIC = "joking/comedic" + CURIOUS = "curious" + PEACEFUL = "peaceful" + SERENE = "serene" + CALM = "calm" + GRATEFUL = "grateful" + AFFECTIONATE = "affectionate" + TRUST = "trust" + SYMPATHETIC = "sympathetic" + ANTICIPATION = "anticipation" + MYSTERIOUS = "mysterious" + MAD = "mad" + OUTRAGED = "outraged" + FRUSTRATED = "frustrated" + AGITATED = "agitated" + THREATENED = "threatened" + DISGUSTED = "disgusted" + CONTEMPT = "contempt" + ENVIOUS = "envious" + SARCASTIC = "sarcastic" + IRONIC = "ironic" + DEJECTED = "dejected" + MELANCHOLIC = "melancholic" + DISAPPOINTED = "disappointed" + HURT = "hurt" + GUILTY = "guilty" + BORED = "bored" + TIRED = "tired" + REJECTED = "rejected" + NOSTALGIC = "nostalgic" + WISTFUL = "wistful" + APOLOGETIC = "apologetic" + HESITANT = "hesitant" + INSECURE = "insecure" + CONFUSED = "confused" + RESIGNED = "resigned" + ANXIOUS = "anxious" + PANICKED = "panicked" + ALARMED = "alarmed" + PROUD = "proud" + CONFIDENT = "confident" + DISTANT = "distant" + SKEPTICAL = "skeptical" + CONTEMPLATIVE = "contemplative" + DETERMINED = "determined" + + class CartesiaTTSService(AudioContextWordTTSService): """Cartesia TTS service with WebSocket streaming and word timestamps. @@ -182,6 +249,10 @@ class CartesiaTTSService(AudioContextWordTTSService): container: Audio container format. params: Additional input parameters for voice customization. text_aggregator: Custom text aggregator for processing input text. + + .. deprecated:: 0.0.95 + Use an LLMTextProcessor before the TTSService for custom text aggregation. + aggregate_sentences: Whether to aggregate sentences within the TTSService. **kwargs: Additional arguments passed to the parent service. """ @@ -200,10 +271,18 @@ class CartesiaTTSService(AudioContextWordTTSService): push_text_frames=False, pause_frame_processing=True, sample_rate=sample_rate, - text_aggregator=text_aggregator or SkipTagsAggregator([("", "")]), + text_aggregator=text_aggregator, **kwargs, ) + if not text_aggregator: + # Always skip tags added for spelled-out text + # Note: This is primarily to support backwards compatibility. + # The preferred way of taking advantage of Cartesia SSML Tags is + # to use an LLMTextProcessor and/or a text_transformer to identify + # and insert these tags for the purpose of the TTS service alone. + self._text_aggregator = SkipTagsAggregator([("", "")]) + params = params or CartesiaTTSService.InputParams() self._api_key = api_key @@ -257,6 +336,27 @@ class CartesiaTTSService(AudioContextWordTTSService): """ return language_to_cartesia_language(language) + # A set of Cartesia-specific helpers for text transformations + def SPELL(text: str) -> str: + """Wrap text in Cartesia spell tag.""" + return f"{text}" + + def EMOTION_TAG(emotion: CartesiaEmotion) -> str: + """Convenience method to create an emotion tag.""" + return f'' + + def PAUSE_TAG(seconds: float) -> str: + """Convenience method to create a pause tag.""" + return f'' + + def VOLUME_TAG(volume: float) -> str: + """Convenience method to create a volume tag.""" + return f'' + + def SPEED_TAG(speed: float) -> str: + """Convenience method to create a speed tag.""" + return f'' + def _is_cjk_language(self, language: str) -> bool: """Check if the given language is CJK (Chinese, Japanese, Korean). @@ -397,8 +497,7 @@ class CartesiaTTSService(AudioContextWordTTSService): ) await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) self._websocket = None await self._call_event_handler("on_connection_error", f"{e}") @@ -410,8 +509,7 @@ class CartesiaTTSService(AudioContextWordTTSService): logger.debug("Disconnecting from Cartesia") await self._websocket.close() except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) finally: self._context_id = None self._websocket = None @@ -464,13 +562,12 @@ class CartesiaTTSService(AudioContextWordTTSService): ) await self.append_to_audio_context(msg["context_id"], frame) elif msg["type"] == "error": - logger.error(f"{self} error: {msg}") await self.push_frame(TTSStoppedFrame()) await self.stop_all_metrics() - await self.push_error(ErrorFrame(error=f"{self} error: {msg['error']}")) + await self.push_error(error_msg=f"Error: {msg}") self._context_id = None else: - logger.error(f"{self} error, unknown message type: {msg}") + await self.push_error(error_msg=f"Error, unknown message type: {msg}") async def _receive_messages(self): while True: @@ -508,16 +605,14 @@ class CartesiaTTSService(AudioContextWordTTSService): await self._get_websocket().send(msg) await self.start_tts_usage_metrics(text) except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") yield TTSStoppedFrame() await self._disconnect() await self._connect() return yield None except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") class CartesiaHttpTTSService(TTSService): @@ -708,8 +803,7 @@ class CartesiaHttpTTSService(TTSService): async with session.post(url, json=payload, headers=headers) as response: if response.status != 200: error_text = await response.text() - logger.error(f"Cartesia API error: {error_text}") - await self.push_error(ErrorFrame(error=f"Cartesia API error: {error_text}")) + yield ErrorFrame(error=f"Cartesia API error: {error_text}") raise Exception(f"Cartesia API returned status {response.status}: {error_text}") audio_data = await response.read() @@ -725,8 +819,7 @@ class CartesiaHttpTTSService(TTSService): yield frame except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + yield ErrorFrame(error=f"Unknown error occurred: {e}") finally: await self.stop_ttfb_metrics() yield TTSStoppedFrame() diff --git a/src/pipecat/services/deepgram/flux/stt.py b/src/pipecat/services/deepgram/flux/stt.py index a0d45afe4..74e9d1053 100644 --- a/src/pipecat/services/deepgram/flux/stt.py +++ b/src/pipecat/services/deepgram/flux/stt.py @@ -150,7 +150,17 @@ class DeepgramFluxSTTService(WebsocketSTTService): params=params ) """ - super().__init__(sample_rate=sample_rate, **kwargs) + # Note: For DeepgramFluxSTTService, differently from other processes, we need to create + # the _receive_task inside _connect_websocket, because the websocket should only be + # considered connected and ready to send audio once we receive from Flux the message + # which confirms the connection has been established. + # If we try to keep the logic reconnect_on_error, when receiving a message, the + # _receive_task_handler would try to reconnect in case of error, invoking the + # _connect_websocket again and leading to a case where the first _receive_task_handler + # was never destroyed. + # So we can keep it here as false, because inside the method send_with_retry, it will + # already try to reconnect if needed. + super().__init__(sample_rate=sample_rate, reconnect_on_error=False, **kwargs) self._api_key = api_key self._url = url @@ -192,8 +202,7 @@ class DeepgramFluxSTTService(WebsocketSTTService): try: await self._disconnect_websocket() except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) finally: # Reset state only after everything is cleaned up self._websocket = None @@ -235,6 +244,11 @@ class DeepgramFluxSTTService(WebsocketSTTService): additional_headers={"Authorization": f"Token {self._api_key}"}, ) + headers = { + k: v for k, v in self._websocket.response.headers.items() if k.startswith("dg-") + } + logger.debug(f'{self}: Websocket connection initialized: {{"headers": {headers}}}') + # Creating the receiver task if not self._receive_task: self._receive_task = self.create_task( @@ -251,8 +265,7 @@ class DeepgramFluxSTTService(WebsocketSTTService): logger.debug("Connected to Deepgram Flux Websocket") await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) self._websocket = None await self._call_event_handler("on_connection_error", f"{e}") @@ -280,8 +293,7 @@ class DeepgramFluxSTTService(WebsocketSTTService): logger.debug("Disconnecting from Deepgram Flux Websocket") await self._websocket.close() except Exception as e: - logger.error(f"{self} error closing websocket: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Error closing websocket: {e}", exception=e) finally: self._websocket = None await self._call_event_handler("on_disconnected") @@ -291,10 +303,13 @@ class DeepgramFluxSTTService(WebsocketSTTService): This signals to the server that no more audio data will be sent. """ - if self._websocket: - logger.debug("Sending CloseStream message to Deepgram Flux") - message = {"type": "CloseStream"} - await self._websocket.send(json.dumps(message)) + try: + if self._websocket: + logger.debug("Sending CloseStream message to Deepgram Flux") + message = {"type": "CloseStream"} + await self._websocket.send(json.dumps(message)) + except Exception as e: + await self.push_error(error_msg=f"Error sending closeStream: {e}", exception=e) def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -381,16 +396,13 @@ class DeepgramFluxSTTService(WebsocketSTTService): are issues sending the audio data. """ if not self._websocket: - logger.error("Not connected to Deepgram Flux.") - yield ErrorFrame("Not connected to Deepgram Flux.") return try: self._last_stt_time = time.monotonic() await self.send_with_retry(audio, self._report_error) except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") return yield None @@ -467,8 +479,7 @@ class DeepgramFluxSTTService(WebsocketSTTService): # Skip malformed messages continue except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) # Error will be handled inside WebsocketService->_receive_task_handler raise else: diff --git a/src/pipecat/services/deepgram/stt.py b/src/pipecat/services/deepgram/stt.py index 0a4cd7e6e..11be561b1 100644 --- a/src/pipecat/services/deepgram/stt.py +++ b/src/pipecat/services/deepgram/stt.py @@ -233,7 +233,14 @@ class DeepgramSTTService(STTService): ) if not await self._connection.start(options=self._settings, addons=self._addons): - logger.error(f"{self}: unable to connect to Deepgram") + await self.push_error(error_msg=f"Unable to connect to Deepgram") + else: + headers = { + k: v + for k, v in self._connection._socket.response.headers.items() + if k.startswith("dg-") + } + logger.debug(f'{self}: Websocket connection initialized: {{"headers": {headers}}}') async def _disconnect(self): if await self._connection.is_connected(): @@ -256,7 +263,7 @@ class DeepgramSTTService(STTService): async def _on_error(self, *args, **kwargs): error: ErrorResponse = kwargs["error"] logger.warning(f"{self} connection error, will retry: {error}") - await self.push_error(ErrorFrame(error=f"{error}")) + await self.push_error(error_msg=f"{error}") await self.stop_all_metrics() # NOTE(aleix): we don't disconnect (i.e. call finish on the connection) # because this triggers more errors internally in the Deepgram SDK. So, diff --git a/src/pipecat/services/deepgram/stt_sagemaker.py b/src/pipecat/services/deepgram/stt_sagemaker.py new file mode 100644 index 000000000..cc765f83a --- /dev/null +++ b/src/pipecat/services/deepgram/stt_sagemaker.py @@ -0,0 +1,444 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Deepgram speech-to-text service for AWS SageMaker. + +This module provides a Pipecat STT service that connects to Deepgram models +deployed on AWS SageMaker endpoints. Uses HTTP/2 bidirectional streaming for +low-latency real-time transcription with support for interim results, multiple +languages, and various Deepgram features. +""" + +import asyncio +import json +from typing import AsyncGenerator, Optional + +from loguru import logger + +from pipecat.frames.frames import ( + CancelFrame, + EndFrame, + ErrorFrame, + Frame, + InterimTranscriptionFrame, + StartFrame, + TranscriptionFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame, +) +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.aws.sagemaker.bidi_client import SageMakerBidiClient +from pipecat.services.stt_service import STTService +from pipecat.transcriptions.language import Language +from pipecat.utils.time import time_now_iso8601 +from pipecat.utils.tracing.service_decorators import traced_stt + +try: + from deepgram import LiveOptions +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error( + "In order to use DeepgramSageMakerSTTService, you need to `pip install pipecat-ai[deepgram,sagemaker]`." + ) + raise Exception(f"Missing module: {e}") + + +class DeepgramSageMakerSTTService(STTService): + """Deepgram speech-to-text service for AWS SageMaker. + + Provides real-time speech recognition using Deepgram models deployed on + AWS SageMaker endpoints. Uses HTTP/2 bidirectional streaming for low-latency + transcription with support for interim results, speaker diarization, and + multiple languages. + + Requirements: + + - AWS credentials configured (via environment variables, AWS CLI, or instance metadata) + - A deployed SageMaker endpoint with Deepgram model: https://developers.deepgram.com/docs/deploy-amazon-sagemaker + - Deepgram SDK for LiveOptions configuration + + Example:: + + stt = DeepgramSageMakerSTTService( + endpoint_name="my-deepgram-endpoint", + region="us-east-2", + live_options=LiveOptions( + model="nova-3", + language="en", + interim_results=True, + punctuate=True, + ), + ) + """ + + def __init__( + self, + *, + endpoint_name: str, + region: str, + sample_rate: Optional[int] = None, + live_options: Optional[LiveOptions] = None, + **kwargs, + ): + """Initialize the Deepgram SageMaker STT service. + + Args: + endpoint_name: Name of the SageMaker endpoint with Deepgram model + deployed (e.g., "my-deepgram-nova-3-endpoint"). + region: AWS region where the endpoint is deployed (e.g., "us-east-2"). + sample_rate: Audio sample rate in Hz. If None, uses value from + live_options or defaults to the value from StartFrame. + live_options: Deepgram LiveOptions for detailed configuration. If None, + uses sensible defaults (nova-3 model, English, interim results enabled). + **kwargs: Additional arguments passed to the parent STTService. + """ + sample_rate = sample_rate or (live_options.sample_rate if live_options else None) + super().__init__(sample_rate=sample_rate, **kwargs) + + self._endpoint_name = endpoint_name + self._region = region + + # Create default options similar to DeepgramSTTService + default_options = LiveOptions( + encoding="linear16", + language=Language.EN, + model="nova-3", + channels=1, + interim_results=True, + punctuate=True, + ) + + # Merge with provided options + merged_options = default_options.to_dict() + if live_options: + default_model = default_options.model + merged_options.update(live_options.to_dict()) + # Handle the "None" string bug from deepgram-sdk + if "model" in merged_options and merged_options["model"] == "None": + merged_options["model"] = default_model + + # Convert Language enum to string if needed + if "language" in merged_options and isinstance(merged_options["language"], Language): + merged_options["language"] = merged_options["language"].value + + self.set_model_name(merged_options["model"]) + self._settings = merged_options + + self._client: Optional[SageMakerBidiClient] = None + self._response_task: Optional[asyncio.Task] = None + self._keepalive_task: Optional[asyncio.Task] = None + + def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Deepgram SageMaker service supports metrics generation. + """ + return True + + async def set_model(self, model: str): + """Set the Deepgram model and reconnect. + + Disconnects from the current session, updates the model setting, and + establishes a new connection with the updated model. + + Args: + model: The Deepgram model name to use (e.g., "nova-3"). + """ + await super().set_model(model) + logger.info(f"Switching STT model to: [{model}]") + self._settings["model"] = model + await self._disconnect() + await self._connect() + + async def set_language(self, language: Language): + """Set the recognition language and reconnect. + + Disconnects from the current session, updates the language setting, and + establishes a new connection with the updated language. + + Args: + language: The language to use for speech recognition (e.g., Language.EN, + Language.ES). + """ + logger.info(f"Switching STT language to: [{language}]") + self._settings["language"] = language + await self._disconnect() + await self._connect() + + async def start(self, frame: StartFrame): + """Start the Deepgram SageMaker STT service. + + Args: + frame: The start frame containing initialization parameters. + """ + await super().start(frame) + self._settings["sample_rate"] = self.sample_rate + await self._connect() + + async def stop(self, frame: EndFrame): + """Stop the Deepgram SageMaker STT service. + + Args: + frame: The end frame. + """ + await super().stop(frame) + await self._disconnect() + + async def cancel(self, frame: CancelFrame): + """Cancel the Deepgram SageMaker STT service. + + Args: + frame: The cancel frame. + """ + await super().cancel(frame) + await self._disconnect() + + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Send audio data to Deepgram for transcription. + + Args: + audio: Raw audio bytes to transcribe. + + Yields: + Frame: None (transcription results come via BiDi stream callbacks). + """ + if self._client and self._client.is_active: + try: + await self._client.send_audio_chunk(audio) + except Exception as e: + yield ErrorFrame(error=f"Unknown error occurred: {e}") + yield None + + async def _connect(self): + """Connect to the SageMaker endpoint and start the BiDi session. + + Builds the Deepgram query string from settings, creates the BiDi client, + starts the streaming session, and launches background tasks for processing + responses and sending KeepAlive messages. + """ + logger.debug("Connecting to Deepgram on SageMaker...") + + # Update sample rate in settings + self._settings["sample_rate"] = self.sample_rate + + # Build query string from settings, converting booleans to strings + query_params = {} + for key, value in self._settings.items(): + if value is not None: + # Convert boolean values to lowercase strings for Deepgram API + if isinstance(value, bool): + query_params[key] = str(value).lower() + else: + query_params[key] = str(value) + + query_string = "&".join(f"{k}={v}" for k, v in query_params.items()) + + # Create BiDi client + self._client = SageMakerBidiClient( + endpoint_name=self._endpoint_name, + region=self._region, + model_invocation_path="v1/listen", + model_query_string=query_string, + ) + + try: + # Start the session + await self._client.start_session() + + # Start processing responses in the background + self._response_task = self.create_task(self._process_responses()) + + # Start keepalive task to maintain connection + self._keepalive_task = self.create_task(self._send_keepalive()) + + logger.debug("Connected to Deepgram on SageMaker") + await self._call_event_handler("on_connected") + + except Exception as e: + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) + await self._call_event_handler("on_connection_error", str(e)) + + async def _disconnect(self): + """Disconnect from the SageMaker endpoint. + + Sends a CloseStream message to Deepgram, cancels background tasks + (KeepAlive and response processing), and closes the BiDi session. + Safe to call multiple times. + """ + if self._client and self._client.is_active: + logger.debug("Disconnecting from Deepgram on SageMaker...") + + # Send CloseStream message to Deepgram + try: + await self._client.send_json({"type": "CloseStream"}) + except Exception as e: + logger.warning(f"Failed to send CloseStream message: {e}") + + # Cancel keepalive task + if self._keepalive_task and not self._keepalive_task.done(): + await self.cancel_task(self._keepalive_task) + + # Cancel response processing task + if self._response_task and not self._response_task.done(): + await self.cancel_task(self._response_task) + + # Close the BiDi session + await self._client.close_session() + + logger.debug("Disconnected from Deepgram on SageMaker") + await self._call_event_handler("on_disconnected") + + async def _send_keepalive(self): + """Send periodic KeepAlive messages to maintain the connection. + + Sends a KeepAlive JSON message to Deepgram every 5 seconds while the + connection is active. This prevents the connection from timing out during + periods of silence. + """ + while self._client and self._client.is_active: + await asyncio.sleep(5) + if self._client and self._client.is_active: + try: + await self._client.send_json({"type": "KeepAlive"}) + except Exception as e: + logger.warning(f"Failed to send KeepAlive: {e}") + + async def _process_responses(self): + """Process streaming responses from Deepgram on SageMaker. + + Continuously receives responses from the BiDi stream, decodes the payload, + parses JSON responses from Deepgram, and processes transcription results. + Runs as a background task until the connection is closed or cancelled. + """ + try: + while self._client and self._client.is_active: + result = await self._client.receive_response() + + if result is None: + break + + # Check if this is a PayloadPart with bytes + if hasattr(result, "value") and hasattr(result.value, "bytes_"): + if result.value.bytes_: + response_data = result.value.bytes_.decode("utf-8") + + try: + # Parse JSON response from Deepgram + parsed = json.loads(response_data) + + # Extract and process transcript if available + if "channel" in parsed: + await self._handle_transcript_response(parsed) + + except json.JSONDecodeError: + logger.warning(f"Non-JSON response: {response_data}") + + except asyncio.CancelledError: + logger.debug("Response processor cancelled") + except Exception as e: + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) + finally: + logger.debug("Response processor stopped") + + async def _handle_transcript_response(self, parsed: dict): + """Handle a transcript response from Deepgram. + + Extracts the transcript text, determines if it's final or interim, extracts + language information, and pushes the appropriate frame (TranscriptionFrame + or InterimTranscriptionFrame) downstream. + + Args: + parsed: The parsed JSON response from Deepgram containing channel, + alternatives, transcript, and metadata. + """ + alternatives = parsed.get("channel", {}).get("alternatives", []) + if not alternatives or not alternatives[0].get("transcript"): + return + + transcript = alternatives[0]["transcript"] + if not transcript.strip(): + return + + # Stop TTFB metrics on first transcript + await self.stop_ttfb_metrics() + + is_final = parsed.get("is_final", False) + speech_final = parsed.get("speech_final", False) + + # Extract language if available + language = None + if alternatives[0].get("languages"): + language = alternatives[0]["languages"][0] + language = Language(language) + + if is_final and speech_final: + # Final transcription + await self.push_frame( + TranscriptionFrame( + transcript, + self._user_id, + time_now_iso8601(), + language, + result=parsed, + ) + ) + await self._handle_transcription(transcript, is_final, language) + await self.stop_processing_metrics() + else: + # Interim transcription + await self.push_frame( + InterimTranscriptionFrame( + transcript, + self._user_id, + time_now_iso8601(), + language, + result=parsed, + ) + ) + + @traced_stt + async def _handle_transcription( + self, transcript: str, is_final: bool, language: Optional[Language] = None + ): + """Handle a transcription result with tracing. + + This method is decorated with @traced_stt for observability and tracing + integration. The actual transcription processing is handled by the parent + class and observers. + + Args: + transcript: The transcribed text. + is_final: Whether this is a final transcription result. + language: The detected language of the transcription, if available. + """ + pass + + async def start_metrics(self): + """Start TTFB and processing metrics collection.""" + await self.start_ttfb_metrics() + await self.start_processing_metrics() + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames with Deepgram SageMaker-specific handling. + + Args: + frame: The frame to process. + direction: The direction of frame processing. + """ + await super().process_frame(frame, direction) + + # Start metrics when user starts speaking (if VAD is not provided by Deepgram) + if isinstance(frame, UserStartedSpeakingFrame): + await self.start_metrics() + elif isinstance(frame, UserStoppedSpeakingFrame): + # Send finalize message to Deepgram when user stops speaking + # This tells Deepgram to flush any remaining audio and return final results + if self._client and self._client.is_active: + try: + await self._client.send_json({"type": "Finalize"}) + except Exception as e: + logger.warning(f"Error sending Finalize message: {e}") diff --git a/src/pipecat/services/deepgram/tts.py b/src/pipecat/services/deepgram/tts.py index f75d40b09..fb52b05e8 100644 --- a/src/pipecat/services/deepgram/tts.py +++ b/src/pipecat/services/deepgram/tts.py @@ -10,35 +10,45 @@ This module provides integration with Deepgram's text-to-speech API for generating speech from text using various voice models. """ +import json from typing import AsyncGenerator, Optional import aiohttp from loguru import logger from pipecat.frames.frames import ( + CancelFrame, + EndFrame, ErrorFrame, Frame, + InterruptionFrame, + LLMFullResponseEndFrame, + StartFrame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, ) -from pipecat.services.tts_service import TTSService +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.tts_service import TTSService, WebsocketTTSService from pipecat.utils.tracing.service_decorators import traced_tts try: - from deepgram import DeepgramClient, DeepgramClientOptions, SpeakOptions + from websockets.asyncio.client import connect as websocket_connect + from websockets.protocol import State except ModuleNotFoundError as e: logger.error(f"Exception: {e}") - logger.error("In order to use Deepgram, you need to `pip install pipecat-ai[deepgram]`.") + logger.error( + "In order to use DeepgramWebsocketTTSService, you need to `pip install pipecat-ai[deepgram]`." + ) raise Exception(f"Missing module: {e}") -class DeepgramTTSService(TTSService): - """Deepgram text-to-speech service. +class DeepgramTTSService(WebsocketTTSService): + """Deepgram WebSocket-based text-to-speech service. - Provides text-to-speech synthesis using Deepgram's streaming API. - Supports various voice models and audio encoding formats with - configurable sample rates and quality settings. + Provides real-time text-to-speech synthesis using Deepgram's WebSocket API. + Supports streaming audio generation with interruption handling via the Clear + message for conversational AI use cases. """ def __init__( @@ -46,42 +56,220 @@ class DeepgramTTSService(TTSService): *, api_key: str, voice: str = "aura-2-helena-en", - base_url: str = "", + base_url: str = "wss://api.deepgram.com", sample_rate: Optional[int] = None, encoding: str = "linear16", **kwargs, ): - """Initialize the Deepgram TTS service. + """Initialize the Deepgram WebSocket TTS service. Args: api_key: Deepgram API key for authentication. voice: Voice model to use for synthesis. Defaults to "aura-2-helena-en". - base_url: Custom base URL for Deepgram API. Uses default if empty. + base_url: WebSocket base URL for Deepgram API. Defaults to "wss://api.deepgram.com". sample_rate: Audio sample rate in Hz. If None, uses service default. encoding: Audio encoding format. Defaults to "linear16". - **kwargs: Additional arguments passed to parent TTSService class. + **kwargs: Additional arguments passed to parent InterruptibleTTSService class. """ - super().__init__(sample_rate=sample_rate, **kwargs) + super().__init__( + sample_rate=sample_rate, + pause_frame_processing=True, + push_stop_frames=True, + **kwargs, + ) + self._api_key = api_key + self._base_url = base_url self._settings = { "encoding": encoding, } self.set_voice(voice) - client_options = DeepgramClientOptions(url=base_url) - self._deepgram_client = DeepgramClient(api_key, config=client_options) + self._receive_task = None def can_generate_metrics(self) -> bool: """Check if the service can generate metrics. Returns: - True, as Deepgram TTS service supports metrics generation. + True, as Deepgram WebSocket TTS service supports metrics generation. """ return True + async def start(self, frame: StartFrame): + """Start the Deepgram WebSocket TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ + await super().start(frame) + await self._connect() + + async def stop(self, frame: EndFrame): + """Stop the Deepgram WebSocket TTS service. + + Args: + frame: The end frame. + """ + await super().stop(frame) + await self._disconnect() + + async def cancel(self, frame: CancelFrame): + """Cancel the Deepgram WebSocket TTS service. + + Args: + frame: The cancel frame. + """ + await super().cancel(frame) + await self._disconnect() + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames with special handling for LLM response end. + + Args: + frame: The frame to process. + direction: The direction of frame processing. + """ + await super().process_frame(frame, direction) + + # When the LLM finishes responding, flush any remaining text in Deepgram's buffer + if isinstance(frame, (LLMFullResponseEndFrame, EndFrame)): + await self.flush_audio() + + async def _connect(self): + """Connect to Deepgram WebSocket and start receive task.""" + await self._connect_websocket() + + if self._websocket and not self._receive_task: + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) + + async def _disconnect(self): + """Disconnect from Deepgram WebSocket and clean up tasks.""" + if self._receive_task: + await self.cancel_task(self._receive_task) + self._receive_task = None + + await self._disconnect_websocket() + + async def _connect_websocket(self): + """Connect to Deepgram WebSocket API with configured settings.""" + try: + if self._websocket and self._websocket.state is State.OPEN: + return + + logger.debug("Connecting to Deepgram WebSocket") + + # Build WebSocket URL with query parameters + params = [] + params.append(f"model={self._voice_id}") + params.append(f"encoding={self._settings['encoding']}") + params.append(f"sample_rate={self.sample_rate}") + + url = f"{self._base_url}/v1/speak?{'&'.join(params)}" + + headers = {"Authorization": f"Token {self._api_key}"} + + self._websocket = await websocket_connect(url, additional_headers=headers) + + headers = { + k: v for k, v in self._websocket.response.headers.items() if k.startswith("dg-") + } + logger.debug(f'{self}: Websocket connection initialized: {{"headers": {headers}}}') + + await self._call_event_handler("on_connected") + except Exception as e: + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + self._websocket = None + await self._call_event_handler("on_connection_error", f"{e}") + + async def _disconnect_websocket(self): + """Close WebSocket connection and reset state.""" + try: + await self.stop_all_metrics() + + if self._websocket: + logger.debug("Disconnecting from Deepgram WebSocket") + # Send Close message to gracefully close the connection + await self._websocket.send(json.dumps({"type": "Close"})) + await self._websocket.close() + except Exception as e: + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + finally: + self._websocket = None + await self._call_event_handler("on_disconnected") + + def _get_websocket(self): + """Get active websocket connection or raise exception.""" + if self._websocket: + return self._websocket + raise Exception("Websocket not connected") + + async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection): + """Handle interruption by sending Clear message to Deepgram. + + The Clear message will clear Deepgram's internal text buffer and stop + sending audio, allowing for a new response to be generated. + """ + await super()._handle_interruption(frame, direction) + + # Send Clear message to stop current audio generation + if self._websocket: + try: + clear_msg = {"type": "Clear"} + await self._websocket.send(json.dumps(clear_msg)) + except Exception as e: + logger.error(f"{self} error sending Clear message: {e}") + + async def _receive_messages(self): + """Receive and process messages from Deepgram WebSocket.""" + async for message in self._get_websocket(): + if isinstance(message, bytes): + # Binary message contains audio data + await self.stop_ttfb_metrics() + frame = TTSAudioRawFrame(message, self.sample_rate, 1) + await self.push_frame(frame) + elif isinstance(message, str): + # Text message contains metadata or control messages + try: + msg = json.loads(message) + msg_type = msg.get("type") + + if msg_type == "Metadata": + logger.trace(f"Received metadata: {msg}") + elif msg_type == "Flushed": + logger.trace(f"Received Flushed: {msg}") + # Flushed indicates the end of audio generation for the current buffer + # This happens after flush_audio() is called + elif msg_type == "Cleared": + logger.trace(f"Received Cleared: {msg}") + # Buffer has been cleared after interruption + # TTSStoppedFrame will be sent by the interruption handler + elif msg_type == "Warning": + logger.warning( + f"{self} warning: {msg.get('description', 'Unknown warning')}" + ) + else: + logger.debug(f"Received unknown message type: {msg}") + except json.JSONDecodeError: + logger.error(f"Invalid JSON message: {message}") + + async def flush_audio(self): + """Flush any pending audio synthesis by sending Flush command. + + This should be called when the LLM finishes a complete response to force + generation of audio from Deepgram's internal text buffer. + """ + if self._websocket: + try: + flush_msg = {"type": "Flush"} + await self._websocket.send(json.dumps(flush_msg)) + except Exception as e: + logger.error(f"{self} error sending Flush message: {e}") + @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - """Generate speech from text using Deepgram's TTS API. + """Generate speech from text using Deepgram's WebSocket TTS API. Args: text: The text to synthesize into speech. @@ -91,33 +279,27 @@ class DeepgramTTSService(TTSService): """ logger.debug(f"{self}: Generating TTS [{text}]") - options = SpeakOptions( - model=self._voice_id, - encoding=self._settings["encoding"], - sample_rate=self.sample_rate, - container="none", - ) - try: + # Reconnect if the websocket is closed + if not self._websocket or self._websocket.state is State.CLOSED: + await self._connect() + await self.start_ttfb_metrics() - - response = await self._deepgram_client.speak.asyncrest.v("1").stream_raw( - {"text": text}, options - ) - await self.start_tts_usage_metrics(text) + yield TTSStartedFrame() - async for data in response.aiter_bytes(): - await self.stop_ttfb_metrics() - if data: - yield TTSAudioRawFrame(audio=data, sample_rate=self.sample_rate, num_channels=1) + # Send text message to Deepgram + # Note: We don't send Flush here - that should only be sent when the + # LLM finishes a complete response via flush_audio() + speak_msg = {"type": "Speak", "text": text} + await self._get_websocket().send(json.dumps(speak_msg)) - yield TTSStoppedFrame() + # The audio frames will be handled in _receive_messages + yield None except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") class DeepgramHttpTTSService(TTSService): @@ -227,5 +409,4 @@ class DeepgramHttpTTSService(TTSService): yield TTSStoppedFrame() except Exception as e: - logger.exception(f"{self} exception: {e}") yield ErrorFrame(f"Error getting audio: {str(e)}") diff --git a/src/pipecat/services/elevenlabs/stt.py b/src/pipecat/services/elevenlabs/stt.py index 8cbb40d63..5fa04d1c1 100644 --- a/src/pipecat/services/elevenlabs/stt.py +++ b/src/pipecat/services/elevenlabs/stt.py @@ -351,8 +351,7 @@ class ElevenLabsSTTService(SegmentedSTTService): ) except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") def audio_format_from_sample_rate(sample_rate: int) -> str: @@ -416,6 +415,8 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): Only used when commit_strategy is VAD. None uses ElevenLabs default. min_silence_duration_ms: Minimum silence duration for VAD (50-2000ms). Only used when commit_strategy is VAD. None uses ElevenLabs default. + include_timestamps: Whether to include word-level timestamps in transcripts. + enable_logging: Whether to enable logging on ElevenLabs' side. """ language_code: Optional[str] = None @@ -424,6 +425,8 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): vad_threshold: Optional[float] = None min_speech_duration_ms: Optional[int] = None min_silence_duration_ms: Optional[int] = None + include_timestamps: bool = False + enable_logging: bool = False def __init__( self, @@ -459,6 +462,8 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): self._audio_format = "" # initialized in start() self._receive_task = None + self._settings = {"language": params.language_code} + def can_generate_metrics(self) -> bool: """Check if the service can generate processing metrics. @@ -477,7 +482,13 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): Changing language requires reconnecting to the WebSocket. """ logger.info(f"Switching STT language to: [{language}]") - self._params.language_code = language.value if isinstance(language, Language) else language + new_language = ( + language_to_elevenlabs_language(language) + if isinstance(language, Language) + else language + ) + self._params.language_code = new_language + self._settings["language"] = new_language # Reconnect with new settings await self._disconnect() await self._connect() @@ -586,7 +597,6 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): } await self._websocket.send(json.dumps(message)) except Exception as e: - logger.error(f"Error sending audio: {e}") yield ErrorFrame(f"ElevenLabs Realtime STT error: {str(e)}") yield None @@ -620,10 +630,16 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): if self._params.language_code: params.append(f"language_code={self._params.language_code}") - params.append(f"encoding={self._audio_format}") - params.append(f"sample_rate={self.sample_rate}") + params.append(f"audio_format={self._audio_format}") params.append(f"commit_strategy={self._params.commit_strategy.value}") + # Add optional parameters + if self._params.include_timestamps: + params.append(f"include_timestamps={str(self._params.include_timestamps).lower()}") + + if self._params.enable_logging: + params.append(f"enable_logging={str(self._params.enable_logging).lower()}") + # Add VAD parameters if using VAD commit strategy and values are specified if self._params.commit_strategy == CommitStrategy.VAD: if self._params.vad_silence_threshold_secs is not None: @@ -645,8 +661,9 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): await self._call_event_handler("on_connected") logger.debug("Connected to ElevenLabs Realtime STT") except Exception as e: - logger.error(f"{self}: unable to connect to ElevenLabs Realtime STT: {e}") - await self.push_error(ErrorFrame(f"Connection error: {str(e)}")) + await self.push_error( + error_msg=f"Unable to connect to ElevenLabs Realtime STT: {e}", exception=e + ) async def _disconnect_websocket(self): """Disconnect from ElevenLabs Realtime STT WebSocket.""" @@ -655,7 +672,7 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): logger.debug("Disconnecting from ElevenLabs Realtime STT") await self._websocket.close() except Exception as e: - logger.error(f"{self} error closing websocket: {e}") + await self.push_error(error_msg=f"Error closing websocket: {e}", exception=e) finally: self._websocket = None await self._call_event_handler("on_disconnected") @@ -712,15 +729,20 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): elif message_type == "committed_transcript_with_timestamps": await self._on_committed_transcript_with_timestamps(data) - elif message_type == "input_error": - error_msg = data.get("error", "Unknown input error") - logger.error(f"ElevenLabs input error: {error_msg}") - await self.push_error(ErrorFrame(f"Input error: {error_msg}")) + elif message_type == "error": + error_msg = data.get("error", "Unknown error") + logger.error(f"ElevenLabs error: {error_msg}") + await self.push_error(error_msg=f"Error: {error_msg}") - elif message_type in ["auth_error", "quota_exceeded", "transcriber_error", "error"]: - error_msg = data.get("error", data.get("message", "Unknown error")) - logger.error(f"ElevenLabs error ({message_type}): {error_msg}") - await self.push_error(ErrorFrame(f"{message_type}: {error_msg}")) + elif message_type == "auth_error": + error_msg = data.get("error", "Authentication error") + logger.error(f"ElevenLabs auth error: {error_msg}") + await self.push_error(error_msg=f"Auth error: {error_msg}") + + elif message_type == "quota_exceeded_error": + error_msg = data.get("error", "Quota exceeded") + logger.error(f"ElevenLabs quota exceeded: {error_msg}") + await self.push_error(error_msg=f"Quota exceeded: {error_msg}") else: logger.debug(f"Unknown message type: {message_type}") @@ -765,6 +787,11 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): Args: data: Committed transcript data. """ + # If timestamps are enabled, skip this message and wait for the + # committed_transcript_with_timestamps message which contains all the data + if self._params.include_timestamps: + return + text = data.get("text", "").strip() if not text: return @@ -792,6 +819,18 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): async def _on_committed_transcript_with_timestamps(self, data: dict): """Handle committed transcript with word-level timestamps. + This message is sent when include_timestamps=true. The result data includes: + - text: The transcribed text + - language_code: Detected language (if available) + - words: Array of word objects with timing information: + - text: The word text + - start: Start time in seconds + - end: End time in seconds + - type: "word" or "spacing" + - speaker_id: Speaker identifier (if available) + - logprob: Log probability score (if available) + - characters: Array of character strings (if available) + Args: data: Committed transcript data with timestamps. """ @@ -799,9 +838,24 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): if not text: return - logger.debug(f"Committed transcript with timestamps: [{text}]") - logger.trace(f"Timestamps: {data.get('words', [])}") + await self.stop_ttfb_metrics() + await self.stop_processing_metrics() - # This is sent after the committed_transcript, so we don't need to - # push another TranscriptionFrame, but we could use the timestamps - # for additional processing if needed in the future + # Get language if provided + language = data.get("language_code") + + logger.debug(f"Committed transcript with timestamps: [{text}]") + + await self._handle_transcription(text, True, language) + + # This message is sent after committed_transcript when include_timestamps=true. + # It contains the full transcript data including text and word-level timestamps. + await self.push_frame( + TranscriptionFrame( + text, + self._user_id, + time_now_iso8601(), + language, + result=data, + ) + ) diff --git a/src/pipecat/services/elevenlabs/tts.py b/src/pipecat/services/elevenlabs/tts.py index bbe05f9dc..f7a3f4367 100644 --- a/src/pipecat/services/elevenlabs/tts.py +++ b/src/pipecat/services/elevenlabs/tts.py @@ -160,7 +160,7 @@ def build_elevenlabs_voice_settings( class PronunciationDictionaryLocator(BaseModel): """Locator for a pronunciation dictionary. - Attributes: + Parameters: pronunciation_dictionary_id: The ID of the pronunciation dictionary. version_id: The version ID of the pronunciation dictionary. """ @@ -424,8 +424,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService): json.dumps({"context_id": self._context_id, "close_context": True}) ) except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) self._context_id = None self._started = False @@ -536,9 +535,8 @@ class ElevenLabsTTSService(AudioContextWordTTSService): await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"{self} exception: {e}") self._websocket = None - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) await self._call_event_handler("on_connection_error", f"{e}") async def _disconnect_websocket(self): @@ -553,8 +551,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService): await self._websocket.close() logger.debug("Disconnected from ElevenLabs") except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) finally: self._started = False self._context_id = None @@ -584,8 +581,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService): json.dumps({"context_id": self._context_id, "close_context": True}) ) except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) self._context_id = None self._started = False self._partial_word = "" @@ -735,20 +731,16 @@ class ElevenLabsTTSService(AudioContextWordTTSService): await self._websocket.send(json.dumps(msg)) logger.trace(f"Created new context {self._context_id}") - await self._send_text(text) - await self.start_tts_usage_metrics(text) - else: - await self._send_text(text) + await self._send_text(text) + await self.start_tts_usage_metrics(text) except Exception as e: - logger.error(f"{self} exception: {e}") yield TTSStoppedFrame() - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") self._started = False return yield None except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") class ElevenLabsHttpTTSService(WordTTSService): @@ -1043,7 +1035,6 @@ class ElevenLabsHttpTTSService(WordTTSService): ) as response: if response.status != 200: error_text = await response.text() - logger.error(f"{self} error: {error_text}") yield ErrorFrame(error=f"ElevenLabs API error: {error_text}") return @@ -1091,8 +1082,7 @@ class ElevenLabsHttpTTSService(WordTTSService): logger.warning(f"Failed to parse JSON from stream: {e}") continue except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") continue # After processing all chunks, emit any remaining partial word @@ -1116,8 +1106,7 @@ class ElevenLabsHttpTTSService(WordTTSService): self._previous_text = text except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") finally: await self.stop_ttfb_metrics() # Let the parent class handle TTSStoppedFrame diff --git a/src/pipecat/services/fal/image.py b/src/pipecat/services/fal/image.py index c110ec3be..6bca6701a 100644 --- a/src/pipecat/services/fal/image.py +++ b/src/pipecat/services/fal/image.py @@ -110,7 +110,6 @@ class FalImageGenService(ImageGenService): image_url = response["images"][0]["url"] if response else None if not image_url: - logger.error(f"{self} error: image generation failed") yield ErrorFrame("Image generation failed") return diff --git a/src/pipecat/services/fal/stt.py b/src/pipecat/services/fal/stt.py index 8b84aaeeb..544f6e128 100644 --- a/src/pipecat/services/fal/stt.py +++ b/src/pipecat/services/fal/stt.py @@ -290,5 +290,4 @@ class FalSTTService(SegmentedSTTService): ) except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") diff --git a/src/pipecat/services/fish/tts.py b/src/pipecat/services/fish/tts.py index 5fe129998..7cc9a3611 100644 --- a/src/pipecat/services/fish/tts.py +++ b/src/pipecat/services/fish/tts.py @@ -76,7 +76,7 @@ class FishAudioTTSService(InterruptibleTTSService): api_key: str, reference_id: Optional[str] = None, # This is the voice ID model: Optional[str] = None, # Deprecated - model_id: str = "speech-1.5", + model_id: str = "s1", output_format: FishAudioOutputFormat = "pcm", sample_rate: Optional[int] = None, params: Optional[InputParams] = None, @@ -93,7 +93,7 @@ class FishAudioTTSService(InterruptibleTTSService): The `model` parameter is deprecated and will be removed in version 0.1.0. Use `reference_id` instead to specify the voice model. - model_id: Specify which Fish Audio TTS model to use (e.g. "speech-1.5") + model_id: Specify which Fish Audio TTS model to use (e.g. "s1") output_format: Audio output format. Defaults to "pcm". sample_rate: Audio sample rate. If None, uses default. params: Additional input parameters for voice customization. @@ -228,8 +228,7 @@ class FishAudioTTSService(InterruptibleTTSService): await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) self._websocket = None await self._call_event_handler("on_connection_error", f"{e}") @@ -243,8 +242,7 @@ class FishAudioTTSService(InterruptibleTTSService): await self._websocket.send(ormsgpack.packb(stop_message)) await self._websocket.close() except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) finally: self._request_id = None self._started = False @@ -286,8 +284,7 @@ class FishAudioTTSService(InterruptibleTTSService): continue except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: @@ -323,8 +320,7 @@ class FishAudioTTSService(InterruptibleTTSService): flush_message = {"event": "flush"} await self._get_websocket().send(ormsgpack.packb(flush_message)) except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") yield TTSStoppedFrame() await self._disconnect() await self._connect() @@ -332,5 +328,4 @@ class FishAudioTTSService(InterruptibleTTSService): yield None except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") diff --git a/src/pipecat/services/gladia/stt.py b/src/pipecat/services/gladia/stt.py index 624745293..27b037f36 100644 --- a/src/pipecat/services/gladia/stt.py +++ b/src/pipecat/services/gladia/stt.py @@ -468,8 +468,7 @@ class GladiaSTTService(STTService): break except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) self._connection_active = False if not self._should_reconnect: @@ -559,8 +558,7 @@ class GladiaSTTService(STTService): except websockets.exceptions.ConnectionClosed: logger.debug("Connection closed during keepalive") except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) async def _receive_task_handler(self): try: @@ -623,8 +621,7 @@ class GladiaSTTService(STTService): # Expected when closing the connection pass except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) async def _maybe_reconnect(self) -> bool: """Handle exponential backoff reconnection logic.""" @@ -632,7 +629,9 @@ class GladiaSTTService(STTService): return False self._reconnection_attempts += 1 if self._reconnection_attempts > self._max_reconnection_attempts: - logger.error(f"Max reconnection attempts ({self._max_reconnection_attempts}) reached") + await self.push_error( + error_msg=f"Max reconnection attempts ({self._max_reconnection_attempts}) reached", + ) self._should_reconnect = False return False delay = self._reconnection_delay * (2 ** (self._reconnection_attempts - 1)) diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index 7e0b0f494..19bcda730 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -27,6 +27,7 @@ from pydantic import BaseModel, Field from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.services.gemini_adapter import GeminiLLMAdapter from pipecat.frames.frames import ( + AggregationType, BotStartedSpeakingFrame, BotStoppedSpeakingFrame, CancelFrame, @@ -1174,7 +1175,7 @@ class GeminiLiveLLMService(LLMService): self._connection_task = self.create_task(self._connection_task_handler(config=config)) except Exception as e: - await self.push_error(ErrorFrame(error=f"{self} Initialization error: {e}")) + await self.push_error(error_msg=f"Initialization error: {e}", exception=e) async def _connection_task_handler(self, config: LiveConnectConfig): async with self._client.aio.live.connect(model=self._model_name, config=config) as session: @@ -1251,11 +1252,11 @@ class GeminiLiveLLMService(LLMService): ) if self._consecutive_failures >= MAX_CONSECUTIVE_FAILURES: - logger.error( + error_msg = ( f"Max consecutive failures ({MAX_CONSECUTIVE_FAILURES}) reached, " "treating as fatal error" ) - await self.push_error(ErrorFrame(error=f"{self} Error in receive loop: {error}")) + await self.push_error(error_msg=error_msg, exception=error) return False else: logger.info( @@ -1283,7 +1284,7 @@ class GeminiLiveLLMService(LLMService): self._completed_tool_calls = set() self._disconnecting = False except Exception as e: - logger.error(f"{self} error disconnecting: {e}") + await self.push_error(error_msg=f"Error disconnecting: {e}", exception=e) async def _send_user_audio(self, frame): """Send user audio frame to Gemini Live API.""" @@ -1644,7 +1645,7 @@ class GeminiLiveLLMService(LLMService): await self.push_frame(TTSStartedFrame()) await self.push_frame(LLMFullResponseStartFrame()) - frame = TTSTextFrame(text=text) + frame = TTSTextFrame(text=text, aggregated_by=AggregationType.SENTENCE) # Gemini Live text already includes any necessary inter-chunk spaces frame.includes_inter_frame_spaces = True @@ -1722,6 +1723,8 @@ class GeminiLiveLLMService(LLMService): prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, total_tokens=total_tokens, + cache_read_input_tokens=usage.cached_content_token_count, + reasoning_tokens=usage.thoughts_token_count, ) await self.start_llm_usage_metrics(tokens) @@ -1742,7 +1745,7 @@ class GeminiLiveLLMService(LLMService): # state management, and that exponential backoff for retries can have # cost/stability implications for a service cluster, let's just treat a # send-side error as fatal. - await self.push_error(ErrorFrame(error=f"{self} Send error: {error}", fatal=True)) + await self.push_error(error_msg=f"Send error: {error}") def create_context_aggregator( self, diff --git a/src/pipecat/services/google/image.py b/src/pipecat/services/google/image.py index 5d7a461b3..f92c7bb2b 100644 --- a/src/pipecat/services/google/image.py +++ b/src/pipecat/services/google/image.py @@ -110,7 +110,6 @@ class GoogleImageGenService(ImageGenService): await self.stop_ttfb_metrics() if not response or not response.generated_images: - logger.error(f"{self} error: image generation failed") yield ErrorFrame("Image generation failed") return @@ -128,5 +127,4 @@ class GoogleImageGenService(ImageGenService): yield frame except Exception as e: - logger.error(f"{self} error generating image: {e}") yield ErrorFrame(f"Image generation error: {str(e)}") diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index 883932b76..840b473b2 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -793,7 +793,7 @@ class GoogleLLMService(LLMService): return generation_params.setdefault("thinking_config", {})["thinking_budget"] = 0 except Exception as e: - logger.exception(f"Failed to unset thinking budget: {e}") + logger.error(f"Failed to unset thinking budget: {e}") async def _stream_content( self, params_from_context: GeminiLLMInvocationParams @@ -983,7 +983,7 @@ class GoogleLLMService(LLMService): except DeadlineExceeded: await self._call_event_handler("on_completion_timeout") except Exception as e: - logger.exception(f"{self} exception: {e}") + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) finally: if grounding_metadata and isinstance(grounding_metadata, dict): llm_search_frame = LLMSearchResponseFrame( diff --git a/src/pipecat/services/google/stt.py b/src/pipecat/services/google/stt.py index beb5db740..7f07ed2f0 100644 --- a/src/pipecat/services/google/stt.py +++ b/src/pipecat/services/google/stt.py @@ -774,8 +774,7 @@ class GoogleSTTService(STTService): yield cloud_speech.StreamingRecognizeRequest(audio=audio_data) except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) raise async def _stream_audio(self): @@ -806,15 +805,13 @@ class GoogleSTTService(STTService): break except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) await asyncio.sleep(1) # Brief delay before reconnecting self._stream_start_time = int(time.time() * 1000) except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: """Process an audio chunk for STT transcription. @@ -902,8 +899,7 @@ class GoogleSTTService(STTService): ) raise except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) # Re-raise the exception to let it propagate (e.g. in the case of a # timeout, propagate to _stream_audio to reconnect) raise diff --git a/src/pipecat/services/google/tts.py b/src/pipecat/services/google/tts.py index cf03e2d52..22b15bb56 100644 --- a/src/pipecat/services/google/tts.py +++ b/src/pipecat/services/google/tts.py @@ -737,7 +737,6 @@ class GoogleHttpTTSService(TTSService): yield TTSStoppedFrame() except Exception as e: - logger.error(f"{self} exception: {e}") error_message = f"TTS generation error: {str(e)}" yield ErrorFrame(error=error_message) @@ -996,9 +995,7 @@ class GoogleTTSService(GoogleBaseTTSService): yield frame except Exception as e: - logger.error(f"{self} exception: {e}") - error_message = f"TTS generation error: {str(e)}" - yield ErrorFrame(error=error_message) + await self.push_error(error_msg=f"TTS generation error: {str(e)}", exception=e) class GeminiTTSService(GoogleBaseTTSService): @@ -1248,6 +1245,5 @@ class GeminiTTSService(GoogleBaseTTSService): yield frame except Exception as e: - logger.error(f"{self} exception: {e}") error_message = f"Gemini TTS generation error: {str(e)}" yield ErrorFrame(error=error_message) diff --git a/src/pipecat/services/gradium/__init__.py b/src/pipecat/services/gradium/__init__.py new file mode 100644 index 000000000..d23112945 --- /dev/null +++ b/src/pipecat/services/gradium/__init__.py @@ -0,0 +1,5 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# diff --git a/src/pipecat/services/gradium/stt.py b/src/pipecat/services/gradium/stt.py new file mode 100644 index 000000000..7b8fc2dff --- /dev/null +++ b/src/pipecat/services/gradium/stt.py @@ -0,0 +1,239 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Gradium's speech-to-text service implementation. + +This module provides integration with Gradium's real-time speech-to-text +WebSocket API for streaming audio transcription. +""" + +import base64 +import json +from typing import AsyncGenerator + +from loguru import logger + +from pipecat.frames.frames import ( + CancelFrame, + EndFrame, + Frame, + StartFrame, + TranscriptionFrame, +) +from pipecat.services.stt_service import WebsocketSTTService +from pipecat.transcriptions.language import Language +from pipecat.utils.time import time_now_iso8601 +from pipecat.utils.tracing.service_decorators import traced_stt + +try: + from websockets.asyncio.client import connect as websocket_connect + from websockets.protocol import State +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error('In order to use Gradium, you need to `pip install "pipecat-ai[gradium]"`.') + raise Exception(f"Missing module: {e}") + +SAMPLE_RATE = 24000 + + +class GradiumSTTService(WebsocketSTTService): + """Gradium real-time speech-to-text service. + + Provides real-time speech transcription using Gradium's WebSocket API. + Supports both interim and final transcriptions with configurable parameters + for audio processing and connection management. + """ + + def __init__( + self, + *, + api_key: str, + api_endpoint_base_url: str = "wss://eu.api.gradium.ai/api/speech/asr", + json_config: str | None = None, + **kwargs, + ): + """Initialize the Gradium STT service. + + Args: + api_key: Gradium API key for authentication. + api_endpoint_base_url: WebSocket endpoint URL. Defaults to Gradium's streaming endpoint. + json_config: Optional JSON configuration string for additional model settings. + **kwargs: Additional arguments passed to parent STTService class. + """ + super().__init__(sample_rate=SAMPLE_RATE, **kwargs) + + self._api_key = api_key + self._api_endpoint_base_url = api_endpoint_base_url + self._websocket = None + self._json_config = json_config + + self._receive_task = None + + self._audio_buffer = bytearray() + self._chunk_size_ms = 80 + self._chunk_size_bytes = 0 + + def can_generate_metrics(self) -> bool: + """Check if the service can generate metrics. + + Returns: + True if metrics generation is supported. + """ + return True + + async def start(self, frame: StartFrame): + """Start the speech-to-text service. + + Args: + frame: Start frame to begin processing. + """ + await super().start(frame) + self._chunk_size_bytes = int(self._chunk_size_ms * self.sample_rate * 2 / 1000) + await self._connect() + + async def stop(self, frame: EndFrame): + """Stop the speech-to-text service. + + Args: + frame: End frame to stop processing. + """ + await super().stop(frame) + await self._disconnect() + + async def cancel(self, frame: CancelFrame): + """Cancel the speech-to-text service. + + Args: + frame: Cancel frame to abort processing. + """ + await super().cancel(frame) + await self._disconnect() + + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Process audio data for speech-to-text conversion. + + Args: + audio: Raw audio bytes to process. + + Yields: + None (processing handled via WebSocket messages). + """ + self._audio_buffer.extend(audio) + await self.start_ttfb_metrics() + await self.start_processing_metrics() + + while len(self._audio_buffer) >= self._chunk_size_bytes: + chunk = bytes(self._audio_buffer[: self._chunk_size_bytes]) + self._audio_buffer = self._audio_buffer[self._chunk_size_bytes :] + chunk = base64.b64encode(chunk).decode("utf-8") + msg = {"type": "audio", "audio": chunk} + if self._websocket and self._websocket.state is State.OPEN: + await self._websocket.send(json.dumps(msg)) + + yield None + + @traced_stt + async def _trace_transcription(self, transcript: str, is_final: bool, language: Language): + """Record transcription event for tracing.""" + pass + + async def _connect(self): + await self._connect_websocket() + + if self._websocket and not self._receive_task: + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) + + async def _connect_websocket(self): + try: + if self._websocket and self._websocket.state is State.OPEN: + return + ws_url = self._api_endpoint_base_url + headers = { + "x-api-key": self._api_key, + "x-api-source": "pipecat", + } + self._websocket = await websocket_connect( + ws_url, + additional_headers=headers, + ) + await self._call_event_handler("on_connected") + setup_msg = { + "type": "setup", + "input_format": "pcm", + } + if self._json_config is not None: + setup_msg["json_config"] = self._json_config + await self._websocket.send(json.dumps(setup_msg)) + ready_msg = await self._websocket.recv() + ready_msg = json.loads(ready_msg) + if ready_msg["type"] == "error": + raise Exception(f"received error {ready_msg['message']}") + if ready_msg["type"] != "ready": + raise Exception(f"unexpected first message type {ready_msg['type']}") + + except Exception as e: + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) + raise + + async def _disconnect(self): + if self._receive_task: + await self.cancel_task(self._receive_task) + self._receive_task = None + + await self._disconnect_websocket() + + async def _disconnect_websocket(self): + try: + if self._websocket and self._websocket.state is State.OPEN: + logger.debug("Disconnecting from Gradium STT") + await self._websocket.close() + except Exception as e: + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) + finally: + self._websocket = None + await self._call_event_handler("on_disconnected") + + def _get_websocket(self): + if self._websocket: + return self._websocket + raise Exception("Websocket not connected") + + async def _process_messages(self): + async for message in self._get_websocket(): + try: + data = json.loads(message) + await self._process_response(data) + except json.JSONDecodeError: + logger.warning(f"Received non-JSON message: {message}") + + async def _receive_messages(self): + while True: + await self._process_messages() + logger.debug(f"{self} Gradium connection was disconnected (timeout?), reconnecting") + await self._connect_websocket() + + async def _process_response(self, msg): + type_ = msg.get("type", "") + if type_ == "text": + await self._handle_text(msg["text"]) + elif type_ == "end_of_stream": + await self._handle_end_of_stream() + elif type_ == "error": + await self.push_error(error_msg=f"Error: {msg}") + + async def _handle_end_of_stream(self): + """Handle termination message.""" + logger.debug("Received end_of_stream message from server") + + async def _handle_text(self, text: str): + """Handle transcription results.""" + await self.push_frame( + TranscriptionFrame( + text, + self._user_id, + time_now_iso8601(), + ) + ) diff --git a/src/pipecat/services/gradium/tts.py b/src/pipecat/services/gradium/tts.py new file mode 100644 index 000000000..8a04f2db6 --- /dev/null +++ b/src/pipecat/services/gradium/tts.py @@ -0,0 +1,315 @@ +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License + +"""Gradium Text-to-Speech service implementation.""" + +import base64 +import json +import uuid +from typing import Any, AsyncGenerator, Mapping, Optional + +from loguru import logger +from pydantic import BaseModel + +from pipecat.frames.frames import ( + CancelFrame, + EndFrame, + ErrorFrame, + Frame, + StartFrame, + TTSAudioRawFrame, + TTSStartedFrame, + TTSStoppedFrame, +) +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.tts_service import InterruptibleWordTTSService +from pipecat.utils.tracing.service_decorators import traced_tts + +try: + from websockets import ConnectionClosedOK + from websockets.asyncio.client import connect as websocket_connect + from websockets.protocol import State +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use Gradium, you need to `pip install pipecat-ai[gradium]`.") + raise Exception(f"Missing module: {e}") + +SAMPLE_RATE = 48000 + + +class GradiumTTSService(InterruptibleWordTTSService): + """Text-to-Speech service using Gradium's websocket API.""" + + class InputParams(BaseModel): + """Configuration parameters for Gradium TTS service. + + Parameters: + temp: Temperature to be used for generation, defaults to 0.6. + """ + + temp: Optional[float] = 0.6 + + def __init__( + self, + *, + api_key: str, + voice_id: str = "YTpq7expH9539ERJ", + url: str = "wss://eu.api.gradium.ai/api/speech/tts", + model: str = "default", + json_config: Optional[str] = None, + params: Optional[InputParams] = None, + **kwargs, + ): + """Initialize the Gradium TTS service. + + Args: + api_key: Gradium API key for authentication. + voice_id: the voice identifier. + url: Gradium websocket API endpoint. + model: Model ID to use for synthesis. + json_config: Optional JSON configuration string for additional model settings. + params: Additional configuration parameters. + **kwargs: Additional arguments passed to parent class. + """ + # Initialize with parent class settings for proper frame handling + super().__init__( + push_stop_frames=True, + pause_frame_processing=True, + sample_rate=SAMPLE_RATE, + **kwargs, + ) + + params = params or GradiumTTSService.InputParams() + + # Store service configuration + self._api_key = api_key + self._url = url + self._voice_id = voice_id + self._json_config = json_config + self._model = model + self._settings = { + "voice_id": voice_id, + "model_name": model, + "output_format": "pcm", + } + + # State tracking + self._receive_task = None + + def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Gradium service supports metrics generation. + """ + return True + + async def set_model(self, model: str): + """Update the TTS model. + + Args: + model: The model name to use for synthesis. + """ + self._model = model + await super().set_model(model) + + async def _update_settings(self, settings: Mapping[str, Any]): + """Update service settings and reconnect if voice changed.""" + prev_voice = self._voice_id + await super()._update_settings(settings) + if not prev_voice == self._voice_id: + self._settings["voice_id"] = self._voice_id + logger.info(f"Switching TTS voice to: [{self._voice_id}]") + await self._disconnect() + await self._connect() + + def _build_msg(self, text: str = "") -> dict: + """Build JSON message for Gradium API.""" + return {"text": text, "type": "text"} + + async def start(self, frame: StartFrame): + """Start the service and establish websocket connection. + + Args: + frame: The start frame containing initialization parameters. + """ + await super().start(frame) + await self._connect() + + async def stop(self, frame: EndFrame): + """Stop the service and close connection. + + Args: + frame: The end frame. + """ + await super().stop(frame) + await self._disconnect() + + async def cancel(self, frame: CancelFrame): + """Cancel current operation and clean up. + + Args: + frame: The cancel frame. + """ + await super().cancel(frame) + await self._disconnect() + + async def _connect(self): + """Establish websocket connection and start receive task.""" + logger.debug(f"{self}: connecting") + + # If the server disconnected, cancel the receive-task so that it can be reset below. + if self._websocket is None or self._websocket.state is not State.OPEN: + if self._receive_task: + await self.cancel_task(self._receive_task) + self._receive_task = None + + await self._connect_websocket() + + if self._websocket and not self._receive_task: + logger.debug(f"{self}: setting receive task") + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) + + async def _disconnect(self): + """Close websocket connection and clean up tasks.""" + logger.debug(f"{self}: disconnecting") + if self._receive_task: + await self.cancel_task(self._receive_task) + self._receive_task = None + + await self._disconnect_websocket() + + async def _connect_websocket(self): + """Connect to Gradium websocket API with configured settings.""" + try: + if self._websocket and self._websocket.state is State.OPEN: + return + + headers = {"x-api-key": self._api_key, "x-api-source": "pipecat"} + self._websocket = await websocket_connect(self._url, additional_headers=headers) + + setup_msg = { + "type": "setup", + "output_format": "pcm", + "voice_id": self._voice_id, + } + if self._json_config is not None: + setup_msg["json_config"] = self._json_config + await self._websocket.send(json.dumps(setup_msg)) + ready_msg = await self._websocket.recv() + ready_msg = json.loads(ready_msg) + if ready_msg["type"] == "error": + raise Exception(f"received error {ready_msg['message']}") + if ready_msg["type"] != "ready": + raise Exception(f"unexpected first message type {ready_msg['type']}") + + await self._call_event_handler("on_connected") + except Exception as e: + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) + self._websocket = None + await self._call_event_handler("on_connection_error", f"{e}") + + async def _disconnect_websocket(self): + """Close websocket connection and reset state.""" + try: + await self.stop_all_metrics() + if self._websocket: + await self._websocket.close() + except Exception as e: + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) + finally: + self._websocket = None + await self._call_event_handler("on_disconnected") + + def _get_websocket(self): + """Get active websocket connection or raise exception.""" + if self._websocket: + return self._websocket + raise Exception("Websocket not connected") + + async def flush_audio(self): + """Flush any pending audio synthesis.""" + if not self._websocket: + return + try: + msg = {"type": "end_of_stream"} + await self._websocket.send(json.dumps(msg)) + except ConnectionClosedOK: + logger.debug(f"{self}: connection closed normally during flush") + except Exception as e: + logger.error(f"{self} exception: {e}") + + async def _receive_messages(self): + """Process incoming websocket messages.""" + # TODO(laurent): This should not be necessary as it should happen when + # receiving the messages but this does not seem to always be the case + # and that may lead to a busy polling loop. + if self._websocket and self._websocket.state is State.CLOSED: + raise ConnectionClosedOK(None, None) + async for message in self._get_websocket(): + msg = json.loads(message) + + if msg["type"] == "audio": + # Process audio chunk + await self.stop_ttfb_metrics() + self.start_word_timestamps() + frame = TTSAudioRawFrame( + audio=base64.b64decode(msg["audio"]), + sample_rate=self.sample_rate, + num_channels=1, + ) + await self.push_frame(frame) + + elif msg["type"] == "text": + await self.add_word_timestamps([(msg["text"], msg["start_s"])]) + elif msg["type"] == "end_of_stream": + await self.push_frame(TTSStoppedFrame()) + await self.stop_all_metrics() + + elif msg["type"] == "error": + await self.push_frame(TTSStoppedFrame()) + await self.stop_all_metrics() + await self.push_error(error_msg=f"Error: {msg['message']}") + + async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): + """Push frame and handle end-of-turn conditions. + + Args: + frame: The frame to push. + direction: The direction to push the frame. + """ + await super().push_frame(frame, direction) + + @traced_tts + async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Gradium's streaming API. + + Args: + text: The text to convert to speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ + _state = self._websocket.state if self._websocket is not None else None + logger.debug(f"{self}: Generating TTS [{text}] {_state}") + try: + if not self._websocket or self._websocket.state is State.CLOSED: + self._websocket = None + await self._connect() + + try: + yield TTSStartedFrame() + + msg = self._build_msg(text=text) + await self._get_websocket().send(json.dumps(msg)) + await self.start_tts_usage_metrics(text) + except Exception as e: + yield ErrorFrame(error=f"Unknown error occurred: {e}") + yield TTSStoppedFrame() + await self._disconnect() + await self._connect() + return + yield None + except Exception as e: + yield ErrorFrame(error=f"Unknown error occurred: {e}") diff --git a/src/pipecat/services/grok/llm.py b/src/pipecat/services/grok/llm.py index 684be2d39..625286a60 100644 --- a/src/pipecat/services/grok/llm.py +++ b/src/pipecat/services/grok/llm.py @@ -123,6 +123,8 @@ class GrokLLMService(OpenAILLMService): self._prompt_tokens = 0 self._completion_tokens = 0 self._total_tokens = 0 + self._cache_read_input_tokens = None + self._reasoning_tokens = None self._has_reported_prompt_tokens = False self._is_processing = True @@ -137,6 +139,8 @@ class GrokLLMService(OpenAILLMService): prompt_tokens=self._prompt_tokens, completion_tokens=self._completion_tokens, total_tokens=self._total_tokens, + cache_read_input_tokens=self._cache_read_input_tokens, + reasoning_tokens=self._reasoning_tokens, ) await super().start_llm_usage_metrics(tokens) @@ -149,7 +153,7 @@ class GrokLLMService(OpenAILLMService): Args: tokens: The token usage metrics for the current chunk of processing, - containing prompt_tokens and completion_tokens counts. + containing prompt_tokens, completion_tokens, and optional cached/reasoning tokens. """ # Only accumulate metrics during active processing if not self._is_processing: @@ -164,6 +168,13 @@ class GrokLLMService(OpenAILLMService): if tokens.completion_tokens > self._completion_tokens: self._completion_tokens = tokens.completion_tokens + # Capture cached & reasoning tokens (these typically only appear once per request) + if tokens.cache_read_input_tokens is not None: + self._cache_read_input_tokens = tokens.cache_read_input_tokens + + if tokens.reasoning_tokens is not None: + self._reasoning_tokens = tokens.reasoning_tokens + def create_context_aggregator( self, context: OpenAILLMContext, diff --git a/src/pipecat/services/groq/tts.py b/src/pipecat/services/groq/tts.py index 9026c4c4c..2966137cb 100644 --- a/src/pipecat/services/groq/tts.py +++ b/src/pipecat/services/groq/tts.py @@ -146,7 +146,6 @@ class GroqTTSService(TTSService): bytes = w.readframes(num_frames) yield TTSAudioRawFrame(bytes, frame_rate, channels) except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") yield TTSStoppedFrame() diff --git a/src/pipecat/services/heygen/client.py b/src/pipecat/services/heygen/client.py index e142aab04..ba517090c 100644 --- a/src/pipecat/services/heygen/client.py +++ b/src/pipecat/services/heygen/client.py @@ -179,7 +179,7 @@ class HeyGenClient: await self._task_manager.cancel_task(self._event_task) self._event_task = None except Exception as e: - logger.exception(f"Exception during cleanup: {e}") + logger.error(f"Exception during cleanup: {e}") async def start(self, frame: StartFrame, audio_chunk_size: int) -> None: """Start the client and establish all necessary connections. diff --git a/src/pipecat/services/hume/tts.py b/src/pipecat/services/hume/tts.py index a3a7e9a4c..6571bff0f 100644 --- a/src/pipecat/services/hume/tts.py +++ b/src/pipecat/services/hume/tts.py @@ -8,27 +8,30 @@ import base64 import os from typing import Any, AsyncGenerator, Optional +import httpx from loguru import logger from pydantic import BaseModel +from pipecat import __version__ from pipecat.frames.frames import ( + CancelFrame, + EndFrame, ErrorFrame, Frame, + InterruptionFrame, StartFrame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, ) -from pipecat.services.tts_service import TTSService +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.tts_service import WordTTSService from pipecat.utils.tracing.service_decorators import traced_tts try: from hume import AsyncHumeClient - from hume.tts import ( - FormatPcm, - PostedUtterance, - PostedUtteranceVoiceWithId, - ) + from hume.tts import FormatPcm, PostedUtterance, PostedUtteranceVoiceWithId + from hume.tts.types import TimestampMessage except ModuleNotFoundError as e: # pragma: no cover - import-time guidance logger.error(f"Exception: {e}") logger.error("In order to use Hume, you need to `pip install pipecat-ai[hume]`.") @@ -37,8 +40,14 @@ except ModuleNotFoundError as e: # pragma: no cover - import-time guidance HUME_SAMPLE_RATE = 48_000 # Hume TTS streams at 48 kHz +# Tracking headers for Hume API requests +DEFAULT_HEADERS = { + "X-Hume-Client-Name": "pipecat", + "X-Hume-Client-Version": __version__, +} -class HumeTTSService(TTSService): + +class HumeTTSService(WordTTSService): """Hume Octave Text-to-Speech service. Streams PCM audio via Hume's HTTP output streaming (JSON chunks) endpoint @@ -48,6 +57,7 @@ class HumeTTSService(TTSService): - Generates speech from text using Hume TTS. - Streams PCM audio. + - Supports word-level timestamps for precise audio-text synchronization. - Supports dynamic updates of voice and synthesis parameters at runtime. - Provides metrics for Time To First Byte (TTFB) and TTS usage. """ @@ -92,9 +102,19 @@ class HumeTTSService(TTSService): f"Hume TTS streams at {HUME_SAMPLE_RATE} Hz; configured sample_rate={sample_rate}" ) - super().__init__(sample_rate=sample_rate, **kwargs) + # WordTTSService sets push_text_frames=False by default, which we want + super().__init__( + sample_rate=sample_rate, + push_text_frames=False, + push_stop_frames=True, + **kwargs, + ) - self._client = AsyncHumeClient(api_key=api_key) + # Create a custom httpx.AsyncClient with tracking headers + # Headers are included in all requests made by the Hume SDK + self._http_client = httpx.AsyncClient(headers=DEFAULT_HEADERS) + + self._client = AsyncHumeClient(api_key=api_key, httpx_client=self._http_client) self._params = params or HumeTTSService.InputParams() # Store voice in the base class (mirrors other services) @@ -102,6 +122,10 @@ class HumeTTSService(TTSService): self._audio_bytes = b"" + # Track cumulative time for word timestamps across utterances + self._cumulative_time = 0.0 + self._started = False + def can_generate_metrics(self) -> bool: """Can generate metrics. @@ -117,6 +141,47 @@ class HumeTTSService(TTSService): frame: The start frame. """ await super().start(frame) + self._reset_state() + + def _reset_state(self): + """Reset internal state variables.""" + self._cumulative_time = 0.0 + self._started = False + + async def stop(self, frame: EndFrame) -> None: + """Stop the service and cleanup resources. + + Args: + frame: The end frame. + """ + await super().stop(frame) + if hasattr(self, "_http_client") and self._http_client: + await self._http_client.aclose() + + async def cancel(self, frame: CancelFrame) -> None: + """Cancel the service and cleanup resources. + + Args: + frame: The cancel frame. + """ + await super().cancel(frame) + if hasattr(self, "_http_client") and self._http_client: + await self._http_client.aclose() + + async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): + """Push a frame and handle state changes. + + Args: + frame: The frame to push. + direction: The direction to push the frame. + """ + await super().push_frame(frame, direction) + if isinstance(frame, (InterruptionFrame, TTSStoppedFrame)): + # Reset timing on interruption or stop + self._reset_state() + + if isinstance(frame, TTSStoppedFrame): + await self.add_word_timestamps([("Reset", 0)]) async def update_setting(self, key: str, value: Any) -> None: """Runtime updates via `TTSUpdateSettingsFrame`. @@ -133,7 +198,7 @@ class HumeTTSService(TTSService): if key_l == "voice_id": self.set_voice(str(value)) - logger.info(f"HumeTTSService voice_id set to: {self.voice}") + logger.debug(f"HumeTTSService voice_id set to: {self.voice}") elif key_l == "description": self._params.description = None if value is None else str(value) elif key_l == "speed": @@ -146,7 +211,7 @@ class HumeTTSService(TTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - """Generate speech from text using Hume TTS. + """Generate speech from text using Hume TTS with word timestamps. Args: text: The text to be synthesized. @@ -177,7 +242,12 @@ class HumeTTSService(TTSService): await self.start_ttfb_metrics() await self.start_tts_usage_metrics(text) - yield TTSStartedFrame() + + # Start TTS sequence if not already started + if not self._started: + self.start_word_timestamps() + yield TTSStartedFrame() + self._started = True try: # Instant mode is always enabled here (not user-configurable) @@ -188,23 +258,50 @@ class HumeTTSService(TTSService): # Use version "2" by default if no description is provided # Version "1" is needed when description is used version = "1" if self._params.description is not None else "2" + + # Track the duration of this utterance based on the last timestamp + utterance_duration = 0.0 + async for chunk in self._client.tts.synthesize_json_streaming( utterances=[utterance], format=pcm_fmt, instant_mode=True, version=version, + include_timestamp_types=["word"], # Request word-level timestamps ): + # Process audio chunks audio_b64 = getattr(chunk, "audio", None) - if not audio_b64: - continue + if audio_b64: + await self.stop_ttfb_metrics() + pcm_bytes = base64.b64decode(audio_b64) + self._audio_bytes += pcm_bytes - pcm_bytes = base64.b64decode(audio_b64) - self._audio_bytes += pcm_bytes + # Buffer audio until we have enough to avoid glitches + if len(self._audio_bytes) >= self.chunk_size: + frame = TTSAudioRawFrame( + audio=self._audio_bytes, + sample_rate=self.sample_rate, + num_channels=1, + ) + yield frame + self._audio_bytes = b"" - # Buffer audio until we have enough to avoid glitches - if len(self._audio_bytes) < self.chunk_size: - continue + # Process timestamp messages + if isinstance(chunk, TimestampMessage): + timestamp = chunk.timestamp + if timestamp.type == "word": + # Convert milliseconds to seconds and add cumulative offset + word_start_time = self._cumulative_time + (timestamp.time.begin / 1000.0) + word_end_time = self._cumulative_time + (timestamp.time.end / 1000.0) + # Track the maximum end time for this utterance + utterance_duration = max(utterance_duration, word_end_time) + + # Add word timestamp + await self.add_word_timestamps([(timestamp.text, word_start_time)]) + + # Flush any remaining audio bytes + if self._audio_bytes: frame = TTSAudioRawFrame( audio=self._audio_bytes, sample_rate=self.sample_rate, @@ -215,10 +312,13 @@ class HumeTTSService(TTSService): self._audio_bytes = b"" + # Update cumulative time for next utterance + if utterance_duration > 0: + self._cumulative_time = utterance_duration + except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) finally: # Ensure TTFB timer is stopped even on early failures await self.stop_ttfb_metrics() - yield TTSStoppedFrame() + # Let the parent class handle TTSStoppedFrame via push_stop_frames diff --git a/src/pipecat/services/inworld/tts.py b/src/pipecat/services/inworld/tts.py index dc2282b91..6652fab72 100644 --- a/src/pipecat/services/inworld/tts.py +++ b/src/pipecat/services/inworld/tts.py @@ -146,6 +146,8 @@ class InworldTTSService(TTSService): Parameters: temperature: Voice temperature control for synthesis variability (e.g., 1.1). Valid range: [0, 2]. Higher values increase variability. + speaking_rate: Speaking speed control (range: [0.5, 1.5]). Defaults to 1.0 when + unset. Note: Language is automatically inferred from the input text by Inworld's TTS models, @@ -153,6 +155,7 @@ class InworldTTSService(TTSService): """ temperature: Optional[float] = None # optional temperature control (range: [0, 2]) + speaking_rate: Optional[float] = None # optional speaking rate control (range: [0.5, 1.5]) def __init__( self, @@ -198,6 +201,7 @@ class InworldTTSService(TTSService): - Other formats as supported by Inworld API params: Optional input parameters for additional configuration. Use this to specify: - temperature: Voice temperature control for variability (range: [0, 2], e.g., 1.1, optional) + - speaking_rate: Set desired speaking speed (range: [0.5, 1.5], optional) Language is automatically inferred from input text. **kwargs: Additional arguments passed to the parent TTSService class. @@ -228,15 +232,18 @@ class InworldTTSService(TTSService): self._settings = { "voiceId": voice_id, # Voice selection from direct parameter "modelId": model, # TTS model selection from direct parameter - "audio_config": { # Audio format configuration - "audio_encoding": encoding, # Format: LINEAR16, MP3, etc. - "sample_rate_hertz": 0, # Will be set in start() from parent service + "audioConfig": { # Audio format configuration + "audioEncoding": encoding, # Format: LINEAR16, MP3, etc. + "sampleRateHertz": 0, # Will be set in start() from parent service }, } # Add optional temperature parameter if provided (valid range: [0, 2]) if params and params.temperature is not None: self._settings["temperature"] = params.temperature + # Add optional speaking rate if provided (valid range: [0.5, 1.5]) + if params and params.speaking_rate is not None: + self._settings["audioConfig"]["speakingRate"] = params.speaking_rate # Register voice and model with parent service for metrics and tracking self.set_voice(voice_id) # Used for logging and metrics @@ -257,7 +264,7 @@ class InworldTTSService(TTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["audio_config"]["sample_rate_hertz"] = self.sample_rate + self._settings["audioConfig"]["sampleRateHertz"] = self.sample_rate async def stop(self, frame: EndFrame): """Stop the Inworld TTS service. @@ -323,9 +330,7 @@ class InworldTTSService(TTSService): "text": text, # Text to synthesize "voiceId": self._settings["voiceId"], # Voice selection (Ashley, Hades, etc.) "modelId": self._settings["modelId"], # TTS model (inworld-tts-1) - "audio_config": self._settings[ - "audio_config" - ], # Audio format settings (LINEAR16, 48kHz) + "audioConfig": self._settings["audioConfig"], # Audio format settings (LINEAR16, 48kHz) } # Add optional temperature parameter if configured (valid range: [0, 2]) @@ -392,8 +397,7 @@ class InworldTTSService(TTSService): # STEP 7: ERROR HANDLING # ================================================================================ # Log any unexpected errors and notify the pipeline - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) finally: # ================================================================================ # STEP 8: CLEANUP AND COMPLETION @@ -508,7 +512,7 @@ class InworldTTSService(TTSService): # Extract the base64-encoded audio content from response if "audioContent" not in response_data: logger.error("No audioContent in Inworld API response") - await self.push_error(ErrorFrame(error="No audioContent in response")) + yield ErrorFrame(error="No audioContent in response") return # ================================================================================ diff --git a/src/pipecat/services/llm_service.py b/src/pipecat/services/llm_service.py index 272df1762..6e5355263 100644 --- a/src/pipecat/services/llm_service.py +++ b/src/pipecat/services/llm_service.py @@ -173,16 +173,17 @@ class LLMService(AIService): run_in_parallel: Whether to run function calls in parallel or sequentially. Defaults to True. **kwargs: Additional arguments passed to the parent AIService. + """ super().__init__(**kwargs) self._run_in_parallel = run_in_parallel self._start_callbacks = {} self._adapter = self.adapter_class() self._functions: Dict[Optional[str], FunctionCallRegistryItem] = {} - self._function_call_tasks: Dict[asyncio.Task, FunctionCallRunnerItem] = {} + self._function_call_tasks: Dict[Optional[asyncio.Task], FunctionCallRunnerItem] = {} self._sequential_runner_task: Optional[asyncio.Task] = None self._tracing_enabled: bool = False - self._skip_tts: bool = False + self._skip_tts: Optional[bool] = None self._register_event_handler("on_function_calls_started") self._register_event_handler("on_completion_timeout") @@ -293,7 +294,8 @@ class LLMService(AIService): direction: The direction of frame pushing. """ if isinstance(frame, (LLMTextFrame, LLMFullResponseStartFrame, LLMFullResponseEndFrame)): - frame.skip_tts = self._skip_tts + if self._skip_tts is not None: + frame.skip_tts = self._skip_tts await super().push_frame(frame, direction) @@ -435,6 +437,7 @@ class LLMService(AIService): await self.broadcast_frame(FunctionCallsStartedFrame, function_calls=function_calls) + runner_items = [] for function_call in function_calls: if function_call.function_name in self._functions.keys(): item = self._functions[function_call.function_name] @@ -446,28 +449,20 @@ class LLMService(AIService): ) continue - runner_item = FunctionCallRunnerItem( - registry_item=item, - function_name=function_call.function_name, - tool_call_id=function_call.tool_call_id, - arguments=function_call.arguments, - context=function_call.context, + runner_items.append( + FunctionCallRunnerItem( + registry_item=item, + function_name=function_call.function_name, + tool_call_id=function_call.tool_call_id, + arguments=function_call.arguments, + context=function_call.context, + ) ) - if self._run_in_parallel: - task = self.create_task(self._run_function_call(runner_item)) - self._function_call_tasks[task] = runner_item - task.add_done_callback(self._function_call_task_finished) - else: - await self._sequential_runner_queue.put(runner_item) - - async def _call_start_function( - self, context: OpenAILLMContext | LLMContext, function_name: str - ): - if function_name in self._start_callbacks.keys(): - await self._start_callbacks[function_name](function_name, self, context) - elif None in self._start_callbacks.keys(): - return await self._start_callbacks[None](function_name, self, context) + if self._run_in_parallel: + await self._run_parallel_function_calls(runner_items) + else: + await self._run_sequential_function_calls(runner_items) async def request_image_frame( self, @@ -540,6 +535,27 @@ class LLMService(AIService): await task del self._function_call_tasks[task] + async def _run_parallel_function_calls(self, runner_items: Sequence[FunctionCallRunnerItem]): + tasks = [] + for runner_item in runner_items: + task = self.create_task(self._run_function_call(runner_item)) + tasks.append(task) + self._function_call_tasks[task] = runner_item + task.add_done_callback(self._function_call_task_finished) + + async def _run_sequential_function_calls(self, runner_items: Sequence[FunctionCallRunnerItem]): + # Enqueue all function calls for background execution. + for runner_item in runner_items: + await self._sequential_runner_queue.put(runner_item) + + async def _call_start_function( + self, context: OpenAILLMContext | LLMContext, function_name: str + ): + if function_name in self._start_callbacks.keys(): + await self._start_callbacks[function_name](function_name, self, context) + elif None in self._start_callbacks.keys(): + return await self._start_callbacks[None](function_name, self, context) + async def _run_function_call(self, runner_item: FunctionCallRunnerItem): if runner_item.function_name in self._functions.keys(): item = self._functions[runner_item.function_name] @@ -623,20 +639,19 @@ class LLMService(AIService): name = runner_item.function_name tool_call_id = runner_item.tool_call_id - # We remove the callback because we are going to cancel the task - # now, otherwise we will be removing it from the set while we - # are iterating. - task.remove_done_callback(self._function_call_task_finished) - logger.debug(f"{self} Cancelling function call [{name}:{tool_call_id}]...") - await self.cancel_task(task) + if task: + # We remove the callback because we are going to cancel the + # task next, otherwise we will be removing it from the set + # while we are iterating. + task.remove_done_callback(self._function_call_task_finished) + await self.cancel_task(task) + cancelled_tasks.add(task) frame = FunctionCallCancelFrame(function_name=name, tool_call_id=tool_call_id) await self.push_frame(frame) - cancelled_tasks.add(task) - logger.debug(f"{self} Function call [{name}:{tool_call_id}] has been cancelled") # Remove all cancelled tasks from our set. diff --git a/src/pipecat/services/lmnt/tts.py b/src/pipecat/services/lmnt/tts.py index ebcad0f20..25de40c4b 100644 --- a/src/pipecat/services/lmnt/tts.py +++ b/src/pipecat/services/lmnt/tts.py @@ -214,8 +214,7 @@ class LmntTTSService(InterruptibleTTSService): await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) self._websocket = None await self._call_event_handler("on_connection_error", f"{e}") @@ -231,8 +230,7 @@ class LmntTTSService(InterruptibleTTSService): # await self._websocket.send(json.dumps({"eof": True})) await self._websocket.close() except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Error disconnecting from LMNT: {e}", exception=e) finally: self._started = False self._websocket = None @@ -266,10 +264,9 @@ class LmntTTSService(InterruptibleTTSService): try: msg = json.loads(message) if "error" in msg: - logger.error(f"{self} error: {msg['error']}") await self.push_frame(TTSStoppedFrame()) await self.stop_all_metrics() - await self.push_error(ErrorFrame(error=f"{self} error: {msg['error']}")) + await self.push_error(error_msg=f"Error: {msg['error']}") return except json.JSONDecodeError: logger.error(f"Invalid JSON message: {message}") @@ -302,13 +299,11 @@ class LmntTTSService(InterruptibleTTSService): await self._get_websocket().send(json.dumps({"flush": True})) await self.start_tts_usage_metrics(text) except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") yield TTSStoppedFrame() await self._disconnect() await self._connect() return yield None except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") diff --git a/src/pipecat/services/mcp_service.py b/src/pipecat/services/mcp_service.py index ccfb47adb..8760c439c 100644 --- a/src/pipecat/services/mcp_service.py +++ b/src/pipecat/services/mcp_service.py @@ -7,7 +7,7 @@ """MCP (Model Context Protocol) client for integrating external tools with LLMs.""" import json -from typing import Any, Dict, List, TypeAlias +from typing import Any, Callable, Dict, List, Optional, TypeAlias from loguru import logger @@ -46,17 +46,24 @@ class MCPClient(BaseObject): def __init__( self, server_params: ServerParameters, + tools_filter: Optional[List[str]] = None, + tools_output_filters: Optional[Dict[str, Callable[[Any], Any]]] = None, **kwargs, ): """Initialize the MCP client with server parameters. Args: server_params: Server connection parameters (stdio or SSE). + tools_filter: Optional list of tool names to register. If None, all tools are registered. + tools_output_filters: Optional dict mapping tool names to filter functions that process tool outputs. + Each filter function receives the raw tool output (any type) and returns the processed output (any type). **kwargs: Additional arguments passed to the parent BaseObject. """ super().__init__(**kwargs) self._server_params = server_params self._session = ClientSession + self._tools_filter = tools_filter + self._tools_output_filters = tools_output_filters or {} if isinstance(server_params, StdioServerParameters): self._client = stdio_client @@ -176,7 +183,6 @@ class MCPClient(BaseObject): except Exception as e: error_msg = f"Error calling mcp tool {params.function_name}: {str(e)}" logger.error(error_msg) - logger.exception("Full exception details:") await params.result_callback(error_msg) async def _stdio_list_tools(self) -> ToolsSchema: @@ -207,7 +213,6 @@ class MCPClient(BaseObject): except Exception as e: error_msg = f"Error calling mcp tool {params.function_name}: {str(e)}" logger.error(error_msg) - logger.exception("Full exception details:") await params.result_callback(error_msg) async def _streamable_http_list_tools(self) -> ToolsSchema: @@ -246,7 +251,6 @@ class MCPClient(BaseObject): except Exception as e: error_msg = f"Error calling mcp tool {params.function_name}: {str(e)}" logger.error(error_msg) - logger.exception("Full exception details:") await params.result_callback(error_msg) async def _call_tool(self, session, function_name, arguments, result_callback): @@ -267,13 +271,26 @@ class MCPClient(BaseObject): else: # logger.debug(f"Non-text result content: '{content}'") pass - logger.info(f"Tool '{function_name}' completed successfully") - logger.debug(f"Final response: {response}") else: logger.error(f"Error getting content from {function_name} results.") - final_response = response if len(response) else "Sorry, could not call the mcp tool" - await result_callback(final_response) + # Apply output filter if configured for this tool + if function_name in self._tools_output_filters: + try: + response = self._tools_output_filters[function_name](response) + logger.debug(f"Final response (after filter): {response}") + + except Exception: + logger.error(f"Error applying output filter for {function_name}") + response = "" + + if response and len(response) and isinstance(response, str): + logger.info(f"Tool '{function_name}' completed successfully") + logger.debug(f"Final response: {response}") + else: + response = "Sorry, could not call the mcp tool" + + await result_callback(response) async def _list_tools_helper(self, session): available_tools = await session.list_tools() @@ -286,6 +303,12 @@ class MCPClient(BaseObject): for tool in available_tools.tools: tool_name = tool.name + + # Apply tools filter if configured + if self._tools_filter and tool_name not in self._tools_filter: + logger.debug(f"Skipping tool '{tool_name}' - not in allowed tools list") + continue + logger.debug(f"Processing tool: {tool_name}") logger.debug(f"Tool description: {tool.description}") @@ -302,7 +325,6 @@ class MCPClient(BaseObject): except Exception as e: logger.error(f"Failed to read tool '{tool_name}': {str(e)}") - logger.exception("Full exception details:") continue logger.debug(f"Completed reading {len(tool_schemas)} tools") diff --git a/src/pipecat/services/mem0/memory.py b/src/pipecat/services/mem0/memory.py index a80398728..66256496c 100644 --- a/src/pipecat/services/mem0/memory.py +++ b/src/pipecat/services/mem0/memory.py @@ -253,8 +253,9 @@ class Mem0MemoryService(FrameProcessor): # Otherwise, pass the enhanced context frame downstream await self.push_frame(frame) except Exception as e: - logger.error(f"Error processing with Mem0: {str(e)}") - await self.push_frame(ErrorFrame(f"Error processing with Mem0: {str(e)}")) + await self.push_error( + error_msg=f"Error processing with Mem0: {str(e)}", exception=e + ) await self.push_frame(frame) # Still pass the original frame through else: # For non-context frames, just pass them through diff --git a/src/pipecat/services/minimax/tts.py b/src/pipecat/services/minimax/tts.py index 05e2dac3b..446a343e7 100644 --- a/src/pipecat/services/minimax/tts.py +++ b/src/pipecat/services/minimax/tts.py @@ -40,24 +40,40 @@ def language_to_minimax_language(language: Language) -> Optional[str]: The corresponding MiniMax language name, or None if not supported. """ LANGUAGE_MAP = { + Language.AF: "Afrikaans", Language.AR: "Arabic", + Language.BG: "Bulgarian", + Language.CA: "Catalan", Language.CS: "Czech", + Language.DA: "Danish", Language.DE: "German", Language.EL: "Greek", Language.EN: "English", Language.ES: "Spanish", + Language.FA: "Persian", # ⚠️ Only supported by speech-2.6-* models Language.FI: "Finnish", + Language.FIL: "Filipino", # ⚠️ Only supported by speech-2.6-* models Language.FR: "French", + Language.HE: "Hebrew", Language.HI: "Hindi", + Language.HR: "Croatian", + Language.HU: "Hungarian", Language.ID: "Indonesian", Language.IT: "Italian", Language.JA: "Japanese", Language.KO: "Korean", + Language.MS: "Malay", + Language.NB: "Norwegian", + Language.NN: "Nynorsk", Language.NL: "Dutch", Language.PL: "Polish", Language.PT: "Portuguese", Language.RO: "Romanian", Language.RU: "Russian", + Language.SK: "Slovak", + Language.SL: "Slovenian", + Language.SV: "Swedish", + Language.TA: "Tamil", # ⚠️ Only supported by speech-2.6-* models Language.TH: "Thai", Language.TR: "Turkish", Language.UK: "Ukrainian", @@ -84,13 +100,22 @@ class MiniMaxHttpTTSService(TTSService): """Configuration parameters for MiniMax TTS. Parameters: - language: Language for TTS generation. + language: Language for TTS generation. Supports 40 languages. + Note: Filipino, Tamil, and Persian require speech-2.6-* models. speed: Speech speed (range: 0.5 to 2.0). volume: Speech volume (range: 0 to 10). pitch: Pitch adjustment (range: -12 to 12). emotion: Emotional tone (options: "happy", "sad", "angry", "fearful", - "disgusted", "surprised", "neutral"). - english_normalization: Whether to apply English text normalization. + "disgusted", "surprised", "calm", "fluent"). + english_normalization: Deprecated; use `text_normalization` instead + + .. deprecated:: 0.0.96 + The `english_normalization` parameter is deprecated and will be removed in a future version. + Use the `text_normalization` parameter instead. + + text_normalization: Enable text normalization (Chinese/English). + latex_read: Enable LaTeX formula reading. + exclude_aggregated_audio: Whether to exclude aggregated audio in final chunk. """ language: Optional[Language] = Language.EN @@ -98,7 +123,10 @@ class MiniMaxHttpTTSService(TTSService): volume: Optional[float] = 1.0 pitch: Optional[int] = 0 emotion: Optional[str] = None - english_normalization: Optional[bool] = None + english_normalization: Optional[bool] = None # Deprecated + text_normalization: Optional[bool] = None + latex_read: Optional[bool] = None + exclude_aggregated_audio: Optional[bool] = None def __init__( self, @@ -120,9 +148,12 @@ class MiniMaxHttpTTSService(TTSService): base_url: API base URL, defaults to MiniMax's T2A endpoint. Global: https://api.minimax.io/v1/t2a_v2 Mainland China: https://api.minimaxi.chat/v1/t2a_v2 + Western United States: https://api-uw.minimax.io/v1/t2a_v2 group_id: MiniMax Group ID to identify project. - model: TTS model name. Defaults to "speech-02-turbo". Options include - "speech-02-hd", "speech-02-turbo", "speech-01-hd", "speech-01-turbo". + model: TTS model name. Defaults to "speech-02-turbo". Options include: + "speech-2.6-hd", "speech-2.6-turbo" (latest, supports Filipino/Tamil/Persian), + "speech-02-hd", "speech-02-turbo", + "speech-01-hd", "speech-01-turbo". voice_id: Voice identifier. Defaults to "Calm_Woman". aiohttp_session: aiohttp.ClientSession for API communication. sample_rate: Output audio sample rate in Hz. If None, uses pipeline default. @@ -176,15 +207,34 @@ class MiniMaxHttpTTSService(TTSService): "disgusted", "surprised", "neutral", + "fluent", ] if params.emotion in supported_emotions: self._settings["voice_setting"]["emotion"] = params.emotion else: - logger.warning(f"Unsupported emotion: {params.emotion}. Using default.") + logger.warning( + f"Unsupported emotion: {params.emotion}. Supported emotions: {supported_emotions}" + ) - # Add english_normalization if provided + # If `english_normalization`, add `text_normalization` and print warning if params.english_normalization is not None: - self._settings["english_normalization"] = params.english_normalization + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Parameter `english_normalization` is deprecated and will be removed in a future version. Use `text_normalization` instead.", + DeprecationWarning, + ) + self._settings["voice_setting"]["text_normalization"] = params.english_normalization + + # Add text_normalization if provided (corrected parameter name) + if params.text_normalization is not None: + self._settings["voice_setting"]["text_normalization"] = params.text_normalization + + # Add latex_read if provided + if params.latex_read is not None: + self._settings["voice_setting"]["latex_read"] = params.latex_read def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -231,7 +281,7 @@ class MiniMaxHttpTTSService(TTSService): """ await super().start(frame) self._settings["audio_setting"]["sample_rate"] = self.sample_rate - logger.debug(f"MiniMax TTS initialized with sample rate: {self.sample_rate}") + logger.debug(f"MiniMax TTS initialized with sample_rate: {self.sample_rate}") @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: @@ -264,7 +314,6 @@ class MiniMaxHttpTTSService(TTSService): ) as response: if response.status != 200: error_message = f"MiniMax TTS error: HTTP {response.status}" - logger.error(error_message) yield ErrorFrame(error=error_message) return @@ -330,16 +379,19 @@ class MiniMaxHttpTTSService(TTSService): num_channels=1, ) except ValueError as e: - logger.error(f"Error converting hex to binary: {e}") + logger.error( + f"Error converting hex to binary: {e}", + ) continue except json.JSONDecodeError as e: - logger.error(f"Error decoding JSON: {e}, data: {data_block[:100]}") + logger.error( + f"Error decoding JSON: {e}, data: {data_block[:100]}", + ) continue except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}", exception=e) finally: await self.stop_ttfb_metrics() yield TTSStoppedFrame() diff --git a/src/pipecat/services/moondream/vision.py b/src/pipecat/services/moondream/vision.py index b7527f76c..e9ce86383 100644 --- a/src/pipecat/services/moondream/vision.py +++ b/src/pipecat/services/moondream/vision.py @@ -110,7 +110,6 @@ class MoondreamService(VisionService): if analysis fails. """ if not self._model: - logger.error(f"{self} error: Moondream model not available ({self.model_name})") yield ErrorFrame("Moondream model not available") return diff --git a/src/pipecat/services/neuphonic/tts.py b/src/pipecat/services/neuphonic/tts.py index 60b0ebcb1..413691ccf 100644 --- a/src/pipecat/services/neuphonic/tts.py +++ b/src/pipecat/services/neuphonic/tts.py @@ -285,8 +285,7 @@ class NeuphonicTTSService(InterruptibleTTSService): await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) self._websocket = None await self._call_event_handler("on_connection_error", f"{e}") @@ -299,8 +298,7 @@ class NeuphonicTTSService(InterruptibleTTSService): logger.debug("Disconnecting from Neuphonic") await self._websocket.close() except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) finally: self._started = False self._websocket = None @@ -365,16 +363,14 @@ class NeuphonicTTSService(InterruptibleTTSService): await self._send_text(text) await self.start_tts_usage_metrics(text) except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") yield TTSStoppedFrame() await self._disconnect() await self._connect() return yield None except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") class NeuphonicHttpTTSService(TTSService): @@ -538,7 +534,6 @@ class NeuphonicHttpTTSService(TTSService): if response.status != 200: error_text = await response.text() error_message = f"Neuphonic API error: HTTP {response.status} - {error_text}" - logger.error(error_message) yield ErrorFrame(error=error_message) return @@ -568,8 +563,7 @@ class NeuphonicHttpTTSService(TTSService): yield TTSAudioRawFrame(audio_bytes, self.sample_rate, 1) except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") # Don't yield error frame for individual message failures continue @@ -577,8 +571,7 @@ class NeuphonicHttpTTSService(TTSService): logger.debug("TTS generation cancelled") raise except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") finally: await self.stop_ttfb_metrics() yield TTSStoppedFrame() diff --git a/src/pipecat/services/nim/llm.py b/src/pipecat/services/nim/llm.py index 07e970521..a59c1da8a 100644 --- a/src/pipecat/services/nim/llm.py +++ b/src/pipecat/services/nim/llm.py @@ -8,98 +8,23 @@ This module provides a service for interacting with NVIDIA's NIM (NVIDIA Inference Microservice) API while maintaining compatibility with the OpenAI-style interface. + +.. deprecated:: 0.0.96 + This module is deprecated. Please NvidiaLLMService from + pipecat.services.nvidia.llm instead. """ -from pipecat.metrics.metrics import LLMTokenUsage -from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.services.openai.llm import OpenAILLMService +import warnings +from pipecat.services.nvidia.llm import NvidiaLLMService -class NimLLMService(OpenAILLMService): - """A service for interacting with NVIDIA's NIM (NVIDIA Inference Microservice) API. +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "NimLLMService from pipecat.services.nim.llm is deprecated. " + "Please use NvidiaLLMService from pipecat.services.nvidia.llm instead.", + DeprecationWarning, + stacklevel=2, + ) - This service extends OpenAILLMService to work with NVIDIA's NIM API while maintaining - compatibility with the OpenAI-style interface. It specifically handles the difference - in token usage reporting between NIM (incremental) and OpenAI (final summary). - """ - - def __init__( - self, - *, - api_key: str, - base_url: str = "https://integrate.api.nvidia.com/v1", - model: str = "nvidia/llama-3.1-nemotron-70b-instruct", - **kwargs, - ): - """Initialize the NimLLMService. - - Args: - api_key: The API key for accessing NVIDIA's NIM API. - base_url: The base URL for NIM API. Defaults to "https://integrate.api.nvidia.com/v1". - model: The model identifier to use. Defaults to "nvidia/llama-3.1-nemotron-70b-instruct". - **kwargs: Additional keyword arguments passed to OpenAILLMService. - """ - super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) - # Counters for accumulating token usage metrics - self._prompt_tokens = 0 - self._completion_tokens = 0 - self._total_tokens = 0 - self._has_reported_prompt_tokens = False - self._is_processing = False - - async def _process_context(self, context: OpenAILLMContext | LLMContext): - """Process a context through the LLM and accumulate token usage metrics. - - This method overrides the parent class implementation to handle NVIDIA's - incremental token reporting style, accumulating the counts and reporting - them once at the end of processing. - - Args: - context: The context to process, containing messages and other information - needed for the LLM interaction. - """ - # Reset all counters and flags at the start of processing - self._prompt_tokens = 0 - self._completion_tokens = 0 - self._total_tokens = 0 - self._has_reported_prompt_tokens = False - self._is_processing = True - - try: - await super()._process_context(context) - finally: - self._is_processing = False - # Report final accumulated token usage at the end of processing - if self._prompt_tokens > 0 or self._completion_tokens > 0: - self._total_tokens = self._prompt_tokens + self._completion_tokens - tokens = LLMTokenUsage( - prompt_tokens=self._prompt_tokens, - completion_tokens=self._completion_tokens, - total_tokens=self._total_tokens, - ) - await super().start_llm_usage_metrics(tokens) - - async def start_llm_usage_metrics(self, tokens: LLMTokenUsage): - """Accumulate token usage metrics during processing. - - This method intercepts the incremental token updates from NVIDIA's API - and accumulates them instead of passing each update to the metrics system. - The final accumulated totals are reported at the end of processing. - - Args: - tokens: The token usage metrics for the current chunk of processing, - containing prompt_tokens and completion_tokens counts. - """ - # Only accumulate metrics during active processing - if not self._is_processing: - return - - # Record prompt tokens the first time we see them - if not self._has_reported_prompt_tokens and tokens.prompt_tokens > 0: - self._prompt_tokens = tokens.prompt_tokens - self._has_reported_prompt_tokens = True - - # Update completion tokens count if it has increased - if tokens.completion_tokens > self._completion_tokens: - self._completion_tokens = tokens.completion_tokens +NimLLMService = NvidiaLLMService diff --git a/src/pipecat/services/nvidia/__init__.py b/src/pipecat/services/nvidia/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/pipecat/services/nvidia/llm.py b/src/pipecat/services/nvidia/llm.py new file mode 100644 index 000000000..75bac586b --- /dev/null +++ b/src/pipecat/services/nvidia/llm.py @@ -0,0 +1,105 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""NVIDIA NIM API service implementation. + +This module provides a service for interacting with NVIDIA's NIM (NVIDIA Inference +Microservice) API while maintaining compatibility with the OpenAI-style interface. +""" + +from pipecat.metrics.metrics import LLMTokenUsage +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.openai.llm import OpenAILLMService + + +class NvidiaLLMService(OpenAILLMService): + """A service for interacting with NVIDIA's NIM (NVIDIA Inference Microservice) API. + + This service extends OpenAILLMService to work with NVIDIA's NIM API while maintaining + compatibility with the OpenAI-style interface. It specifically handles the difference + in token usage reporting between NIM (incremental) and OpenAI (final summary). + """ + + def __init__( + self, + *, + api_key: str, + base_url: str = "https://integrate.api.nvidia.com/v1", + model: str = "nvidia/llama-3.1-nemotron-70b-instruct", + **kwargs, + ): + """Initialize the NvidiaLLMService. + + Args: + api_key: The API key for accessing NVIDIA's NIM API. + base_url: The base URL for NIM API. Defaults to "https://integrate.api.nvidia.com/v1". + model: The model identifier to use. Defaults to "nvidia/llama-3.1-nemotron-70b-instruct". + **kwargs: Additional keyword arguments passed to OpenAILLMService. + """ + super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) + # Counters for accumulating token usage metrics + self._prompt_tokens = 0 + self._completion_tokens = 0 + self._total_tokens = 0 + self._has_reported_prompt_tokens = False + self._is_processing = False + + async def _process_context(self, context: OpenAILLMContext | LLMContext): + """Process a context through the LLM and accumulate token usage metrics. + + This method overrides the parent class implementation to handle NVIDIA's + incremental token reporting style, accumulating the counts and reporting + them once at the end of processing. + + Args: + context: The context to process, containing messages and other information + needed for the LLM interaction. + """ + # Reset all counters and flags at the start of processing + self._prompt_tokens = 0 + self._completion_tokens = 0 + self._total_tokens = 0 + self._has_reported_prompt_tokens = False + self._is_processing = True + + try: + await super()._process_context(context) + finally: + self._is_processing = False + # Report final accumulated token usage at the end of processing + if self._prompt_tokens > 0 or self._completion_tokens > 0: + self._total_tokens = self._prompt_tokens + self._completion_tokens + tokens = LLMTokenUsage( + prompt_tokens=self._prompt_tokens, + completion_tokens=self._completion_tokens, + total_tokens=self._total_tokens, + ) + await super().start_llm_usage_metrics(tokens) + + async def start_llm_usage_metrics(self, tokens: LLMTokenUsage): + """Accumulate token usage metrics during processing. + + This method intercepts the incremental token updates from NVIDIA's API + and accumulates them instead of passing each update to the metrics system. + The final accumulated totals are reported at the end of processing. + + Args: + tokens: The token usage metrics for the current chunk of processing, + containing prompt_tokens and completion_tokens counts. + """ + # Only accumulate metrics during active processing + if not self._is_processing: + return + + # Record prompt tokens the first time we see them + if not self._has_reported_prompt_tokens and tokens.prompt_tokens > 0: + self._prompt_tokens = tokens.prompt_tokens + self._has_reported_prompt_tokens = True + + # Update completion tokens count if it has increased + if tokens.completion_tokens > self._completion_tokens: + self._completion_tokens = tokens.completion_tokens diff --git a/src/pipecat/services/nvidia/stt.py b/src/pipecat/services/nvidia/stt.py new file mode 100644 index 000000000..5475ac14d --- /dev/null +++ b/src/pipecat/services/nvidia/stt.py @@ -0,0 +1,663 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""NVIDIA Riva Speech-to-Text service implementations for real-time and batch transcription.""" + +import asyncio +from concurrent.futures import CancelledError as FuturesCancelledError +from typing import AsyncGenerator, List, Mapping, Optional + +from loguru import logger +from pydantic import BaseModel + +from pipecat.frames.frames import ( + CancelFrame, + EndFrame, + ErrorFrame, + Frame, + InterimTranscriptionFrame, + StartFrame, + TranscriptionFrame, +) +from pipecat.services.stt_service import SegmentedSTTService, STTService +from pipecat.transcriptions.language import Language, resolve_language +from pipecat.utils.time import time_now_iso8601 +from pipecat.utils.tracing.service_decorators import traced_stt + +try: + import riva.client + +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use NVIDIA Riva STT, you need to `pip install pipecat-ai[nvidia]`.") + raise Exception(f"Missing module: {e}") + + +def language_to_nvidia_riva_language(language: Language) -> Optional[str]: + """Maps Language enum to NVIDIA Riva ASR language codes. + + Source: + https://docs.nvidia.com/deeplearning/riva/user-guide/docs/asr/asr-riva-build-table.html?highlight=fr%20fr + + Args: + language: Language enum value. + + Returns: + Optional[str]: NVIDIA Riva language code or None if not supported. + """ + LANGUAGE_MAP = { + # Arabic + Language.AR: "ar-AR", + # English + Language.EN: "en-US", # Default to US + Language.EN_US: "en-US", + Language.EN_GB: "en-GB", + # French + Language.FR: "fr-FR", + Language.FR_FR: "fr-FR", + # German + Language.DE: "de-DE", + Language.DE_DE: "de-DE", + # Hindi + Language.HI: "hi-IN", + Language.HI_IN: "hi-IN", + # Italian + Language.IT: "it-IT", + Language.IT_IT: "it-IT", + # Japanese + Language.JA: "ja-JP", + Language.JA_JP: "ja-JP", + # Korean + Language.KO: "ko-KR", + Language.KO_KR: "ko-KR", + # Portuguese + Language.PT: "pt-BR", # Default to Brazilian + Language.PT_BR: "pt-BR", + # Russian + Language.RU: "ru-RU", + Language.RU_RU: "ru-RU", + # Spanish + Language.ES: "es-ES", # Default to Spain + Language.ES_ES: "es-ES", + Language.ES_US: "es-US", # US Spanish + } + + return resolve_language(language, LANGUAGE_MAP, use_base_code=False) + + +class NvidiaSTTService(STTService): + """Real-time speech-to-text service using NVIDIA Riva streaming ASR. + + Provides real-time transcription capabilities using NVIDIA's Riva ASR models + through streaming recognition. Supports interim results and continuous audio + processing for low-latency applications. + """ + + class InputParams(BaseModel): + """Configuration parameters for NVIDIA Riva STT service. + + Parameters: + language: Target language for transcription. Defaults to EN_US. + """ + + language: Optional[Language] = Language.EN_US + + def __init__( + self, + *, + api_key: str, + server: str = "grpc.nvcf.nvidia.com:443", + model_function_map: Mapping[str, str] = { + "function_id": "1598d209-5e27-4d3c-8079-4751568b1081", + "model_name": "parakeet-ctc-1.1b-asr", + }, + sample_rate: Optional[int] = None, + params: Optional[InputParams] = None, + **kwargs, + ): + """Initialize the NVIDIA Riva STT service. + + Args: + api_key: NVIDIA API key for authentication. + server: NVIDIA Riva server address. Defaults to NVIDIA Cloud Function endpoint. + model_function_map: Mapping containing 'function_id' and 'model_name' for the ASR model. + sample_rate: Audio sample rate in Hz. If None, uses pipeline default. + params: Additional configuration parameters for NVIDIA Riva. + **kwargs: Additional arguments passed to STTService. + """ + super().__init__(sample_rate=sample_rate, **kwargs) + + params = params or NvidiaSTTService.InputParams() + + self._api_key = api_key + self._profanity_filter = False + self._automatic_punctuation = True + self._no_verbatim_transcripts = False + self._language_code = params.language + self._boosted_lm_words = None + self._boosted_lm_score = 4.0 + self._start_history = -1 + self._start_threshold = -1.0 + self._stop_history = -1 + self._stop_threshold = -1.0 + self._stop_history_eou = -1 + self._stop_threshold_eou = -1.0 + self._custom_configuration = "" + self._function_id = model_function_map.get("function_id") + + self._settings = { + "language": str(params.language), + "profanity_filter": self._profanity_filter, + "automatic_punctuation": self._automatic_punctuation, + "verbatim_transcripts": not self._no_verbatim_transcripts, + "boosted_lm_words": self._boosted_lm_words, + "boosted_lm_score": self._boosted_lm_score, + } + + self.set_model_name(model_function_map.get("model_name")) + + metadata = [ + ["function-id", self._function_id], + ["authorization", f"Bearer {api_key}"], + ] + auth = riva.client.Auth(None, True, server, metadata) + + self._asr_service = riva.client.ASRService(auth) + + self._queue = None + self._config = None + self._thread_task = None + self._response_task = None + + def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + False - this service does not support metrics generation. + """ + return False + + async def set_model(self, model: str): + """Set the ASR model for transcription. + + Args: + model: Model name to set. + + Note: + Model cannot be changed after initialization. Use model_function_map + parameter in constructor instead. + """ + logger.warning(f"Cannot set model after initialization. Set model and function id like so:") + example = {"function_id": "", "model_name": ""} + logger.warning( + f"{self.__class__.__name__}(api_key=, model_function_map={example})" + ) + + async def start(self, frame: StartFrame): + """Start the NVIDIA Riva STT service and initialize streaming configuration. + + Args: + frame: StartFrame indicating pipeline start. + """ + await super().start(frame) + + if self._config: + return + + config = riva.client.StreamingRecognitionConfig( + config=riva.client.RecognitionConfig( + encoding=riva.client.AudioEncoding.LINEAR_PCM, + language_code=self._language_code, + model="", + max_alternatives=1, + profanity_filter=self._profanity_filter, + enable_automatic_punctuation=self._automatic_punctuation, + verbatim_transcripts=not self._no_verbatim_transcripts, + sample_rate_hertz=self.sample_rate, + audio_channel_count=1, + ), + interim_results=True, + ) + + riva.client.add_word_boosting_to_config( + config, self._boosted_lm_words, self._boosted_lm_score + ) + + riva.client.add_endpoint_parameters_to_config( + config, + self._start_history, + self._start_threshold, + self._stop_history, + self._stop_history_eou, + self._stop_threshold, + self._stop_threshold_eou, + ) + riva.client.add_custom_configuration_to_config(config, self._custom_configuration) + + self._config = config + self._queue = asyncio.Queue() + + if not self._thread_task: + self._thread_task = self.create_task(self._thread_task_handler()) + + if not self._response_task: + self._response_queue = asyncio.Queue() + self._response_task = self.create_task(self._response_task_handler()) + + async def stop(self, frame: EndFrame): + """Stop the NVIDIA Riva STT service and clean up resources. + + Args: + frame: EndFrame indicating pipeline stop. + """ + await super().stop(frame) + await self._stop_tasks() + + async def cancel(self, frame: CancelFrame): + """Cancel the NVIDIA Riva STT service operation. + + Args: + frame: CancelFrame indicating operation cancellation. + """ + await super().cancel(frame) + await self._stop_tasks() + + async def _stop_tasks(self): + if self._thread_task: + await self.cancel_task(self._thread_task) + self._thread_task = None + + if self._response_task: + await self.cancel_task(self._response_task) + self._response_task = None + + def _response_handler(self): + responses = self._asr_service.streaming_response_generator( + audio_chunks=self, + streaming_config=self._config, + ) + for response in responses: + if not response.results: + continue + asyncio.run_coroutine_threadsafe( + self._response_queue.put(response), self.get_event_loop() + ) + + async def _thread_task_handler(self): + try: + self._thread_running = True + await asyncio.to_thread(self._response_handler) + except asyncio.CancelledError: + self._thread_running = False + raise + + @traced_stt + async def _handle_transcription( + self, transcript: str, is_final: bool, language: Optional[Language] = None + ): + """Handle a transcription result with tracing.""" + pass + + async def _handle_response(self, response): + for result in response.results: + if result and not result.alternatives: + continue + + transcript = result.alternatives[0].transcript + if transcript and len(transcript) > 0: + await self.stop_ttfb_metrics() + if result.is_final: + await self.stop_processing_metrics() + await self.push_frame( + TranscriptionFrame( + transcript, + self._user_id, + time_now_iso8601(), + self._language_code, + result=result, + ) + ) + await self._handle_transcription( + transcript=transcript, + is_final=result.is_final, + language=self._language_code, + ) + else: + await self.push_frame( + InterimTranscriptionFrame( + transcript, + self._user_id, + time_now_iso8601(), + self._language_code, + result=result, + ) + ) + + async def _response_task_handler(self): + while True: + response = await self._response_queue.get() + await self._handle_response(response) + self._response_queue.task_done() + + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Process audio data for speech-to-text transcription. + + Args: + audio: Raw audio bytes to transcribe. + + Yields: + None - transcription results are pushed to the pipeline via frames. + """ + await self.start_ttfb_metrics() + await self.start_processing_metrics() + await self._queue.put(audio) + yield None + + def __next__(self) -> bytes: + """Get the next audio chunk for NVIDIA Riva processing. + + Returns: + Audio bytes from the queue. + + Raises: + StopIteration: When the thread is no longer running. + """ + if not self._thread_running: + raise StopIteration + + try: + future = asyncio.run_coroutine_threadsafe(self._queue.get(), self.get_event_loop()) + return future.result() + except FuturesCancelledError: + raise StopIteration + + def __iter__(self): + """Return iterator for audio chunk processing. + + Returns: + Self as iterator. + """ + return self + + +class NvidiaSegmentedSTTService(SegmentedSTTService): + """Speech-to-text service using NVIDIA Riva's offline/batch models. + + By default, his service uses NVIDIA's Riva Canary ASR API to perform speech-to-text + transcription on audio segments. It inherits from SegmentedSTTService to handle + audio buffering and speech detection. + """ + + class InputParams(BaseModel): + """Configuration parameters for NVIDIA Riva segmented STT service. + + Parameters: + language: Target language for transcription. Defaults to EN_US. + profanity_filter: Whether to filter profanity from results. + automatic_punctuation: Whether to add automatic punctuation. + verbatim_transcripts: Whether to return verbatim transcripts. + boosted_lm_words: List of words to boost in language model. + boosted_lm_score: Score boost for specified words. + """ + + language: Optional[Language] = Language.EN_US + profanity_filter: bool = False + automatic_punctuation: bool = True + verbatim_transcripts: bool = False + boosted_lm_words: Optional[List[str]] = None + boosted_lm_score: float = 4.0 + + def __init__( + self, + *, + api_key: str, + server: str = "grpc.nvcf.nvidia.com:443", + model_function_map: Mapping[str, str] = { + "function_id": "ee8dc628-76de-4acc-8595-1836e7e857bd", + "model_name": "canary-1b-asr", + }, + sample_rate: Optional[int] = None, + params: Optional[InputParams] = None, + **kwargs, + ): + """Initialize the NVIDIA Riva segmented STT service. + + Args: + api_key: NVIDIA API key for authentication + server: NVIDIA Riva server address (defaults to NVIDIA Cloud Function endpoint) + model_function_map: Mapping of model name and its corresponding NVIDIA Cloud Function ID + sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate + params: Additional configuration parameters for NVIDIA Riva + **kwargs: Additional arguments passed to SegmentedSTTService + """ + super().__init__(sample_rate=sample_rate, **kwargs) + + params = params or NvidiaSegmentedSTTService.InputParams() + + # Set model name + self.set_model_name(model_function_map.get("model_name")) + + # Initialize NVIDIA Riva settings + self._api_key = api_key + self._server = server + self._function_id = model_function_map.get("function_id") + self._model_name = model_function_map.get("model_name") + + # Store the language as a Language enum and as a string + self._language_enum = params.language or Language.EN_US + self._language = self.language_to_service_language(self._language_enum) or "en-US" + + # Configure transcription parameters + self._profanity_filter = params.profanity_filter + self._automatic_punctuation = params.automatic_punctuation + self._verbatim_transcripts = params.verbatim_transcripts + self._boosted_lm_words = params.boosted_lm_words + self._boosted_lm_score = params.boosted_lm_score + + # Voice activity detection thresholds (use NVIDIA Riva defaults) + self._start_history = -1 + self._start_threshold = -1.0 + self._stop_history = -1 + self._stop_threshold = -1.0 + self._stop_history_eou = -1 + self._stop_threshold_eou = -1.0 + self._custom_configuration = "" + + # Create NVIDIA Riva client + self._config = None + self._asr_service = None + self._settings = {"language": self._language_enum} + + def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert pipecat Language enum to NVIDIA Riva's language code. + + Args: + language: Language enum value. + + Returns: + NVIDIA Riva language code or None if not supported. + """ + return language_to_nvidia_riva_language(language) + + def _initialize_client(self): + """Initialize the NVIDIA Riva ASR client with authentication metadata.""" + if self._asr_service is not None: + return + + # Set up authentication metadata for NVIDIA Cloud Functions + metadata = [ + ["function-id", self._function_id], + ["authorization", f"Bearer {self._api_key}"], + ] + + # Create authenticated client + auth = riva.client.Auth(None, True, self._server, metadata) + self._asr_service = riva.client.ASRService(auth) + + logger.info(f"Initialized NvidiaSegmentedSTTService with model: {self.model_name}") + + def _create_recognition_config(self): + """Create the NVIDIA Riva ASR recognition configuration.""" + # Create base configuration + config = riva.client.RecognitionConfig( + language_code=self._language, # Now using the string, not a tuple + max_alternatives=1, + profanity_filter=self._profanity_filter, + enable_automatic_punctuation=self._automatic_punctuation, + verbatim_transcripts=self._verbatim_transcripts, + ) + + # Add word boosting if specified + if self._boosted_lm_words: + riva.client.add_word_boosting_to_config( + config, self._boosted_lm_words, self._boosted_lm_score + ) + + # Add voice activity detection parameters + riva.client.add_endpoint_parameters_to_config( + config, + self._start_history, + self._start_threshold, + self._stop_history, + self._stop_history_eou, + self._stop_threshold, + self._stop_threshold_eou, + ) + + # Add any custom configuration + if self._custom_configuration: + riva.client.add_custom_configuration_to_config(config, self._custom_configuration) + + return config + + def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True - this service supports metrics generation. + """ + return True + + async def set_model(self, model: str): + """Set the ASR model for transcription. + + Args: + model: Model name to set. + + Note: + Model cannot be changed after initialization. Use model_function_map + parameter in constructor instead. + """ + logger.warning(f"Cannot set model after initialization. Set model and function id like so:") + example = {"function_id": "", "model_name": ""} + logger.warning( + f"{self.__class__.__name__}(api_key=, model_function_map={example})" + ) + + async def start(self, frame: StartFrame): + """Initialize the service when the pipeline starts. + + Args: + frame: StartFrame indicating pipeline start. + """ + await super().start(frame) + self._initialize_client() + self._config = self._create_recognition_config() + + async def set_language(self, language: Language): + """Set the language for the STT service. + + Args: + language: Target language for transcription. + """ + logger.info(f"Switching STT language to: [{language}]") + self._language_enum = language + self._language = self.language_to_service_language(language) or "en-US" + self._settings["language"] = language + + # Update configuration with new language + if self._config: + self._config.language_code = self._language + + @traced_stt + async def _handle_transcription( + self, transcript: str, is_final: bool, language: Optional[Language] = None + ): + """Handle a transcription result with tracing.""" + pass + + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Transcribe an audio segment. + + Args: + audio: Raw audio bytes in WAV format (already converted by base class). + + Yields: + Frame: TranscriptionFrame containing the transcribed text. + """ + try: + await self.start_processing_metrics() + await self.start_ttfb_metrics() + + # Make sure the client is initialized + if self._asr_service is None: + self._initialize_client() + + # Make sure the config is created + if self._config is None: + self._config = self._create_recognition_config() + + # Type assertion to satisfy the IDE + assert self._asr_service is not None, "ASR service not initialized" + assert self._config is not None, "Recognition config not created" + + # Process audio with NVIDIA Riva ASR - explicitly request non-future response + raw_response = self._asr_service.offline_recognize(audio, self._config, future=False) + + await self.stop_ttfb_metrics() + await self.stop_processing_metrics() + + # Process the response - handle different possible return types + try: + # If it's a future-like object, get the result + if hasattr(raw_response, "result"): + response = raw_response.result() + else: + response = raw_response + + # Process transcription results + transcription_found = False + + # Now we can safely check results + # Type hint for the IDE + results = getattr(response, "results", []) + + for result in results: + alternatives = getattr(result, "alternatives", []) + if alternatives: + text = alternatives[0].transcript.strip() + if text: + logger.debug(f"Transcription: [{text}]") + yield TranscriptionFrame( + text, + self._user_id, + time_now_iso8601(), + self._language_enum, + ) + transcription_found = True + + await self._handle_transcription(text, True, self._language_enum) + + if not transcription_found: + logger.debug("No transcription results found in NVIDIA Riva response") + + except AttributeError as ae: + logger.error(f"Unexpected response structure from NVIDIA Riva: {ae}") + yield ErrorFrame(f"Unexpected NVIDIA Riva response format: {str(ae)}") + + except Exception as e: + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") diff --git a/src/pipecat/services/nvidia/tts.py b/src/pipecat/services/nvidia/tts.py new file mode 100644 index 000000000..91eb138e0 --- /dev/null +++ b/src/pipecat/services/nvidia/tts.py @@ -0,0 +1,187 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""NVIDIA Riva text-to-speech service implementation. + +This module provides integration with NVIDIA Riva's TTS services through +gRPC API for high-quality speech synthesis. +""" + +import asyncio +import os +from typing import AsyncGenerator, Mapping, Optional + +from pipecat.utils.tracing.service_decorators import traced_tts + +# Suppress gRPC fork warnings +os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false" + +from loguru import logger +from pydantic import BaseModel + +from pipecat.frames.frames import ( + ErrorFrame, + Frame, + TTSAudioRawFrame, + TTSStartedFrame, + TTSStoppedFrame, +) +from pipecat.services.tts_service import TTSService +from pipecat.transcriptions.language import Language + +try: + import riva.client + +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use NVIDIA Riva TTS, you need to `pip install pipecat-ai[nvidia]`.") + raise Exception(f"Missing module: {e}") + +NVIDIA_TTS_TIMEOUT_SECS = 5 + + +class NvidiaTTSService(TTSService): + """NVIDIA Riva text-to-speech service. + + Provides high-quality text-to-speech synthesis using NVIDIA Riva's + cloud-based TTS models. Supports multiple voices, languages, and + configurable quality settings. + """ + + class InputParams(BaseModel): + """Input parameters for Riva TTS configuration. + + Parameters: + language: Language code for synthesis. Defaults to US English. + quality: Audio quality setting (0-100). Defaults to 20. + """ + + language: Optional[Language] = Language.EN_US + quality: Optional[int] = 20 + + def __init__( + self, + *, + api_key: str, + server: str = "grpc.nvcf.nvidia.com:443", + voice_id: str = "Magpie-Multilingual.EN-US.Aria", + sample_rate: Optional[int] = None, + model_function_map: Mapping[str, str] = { + "function_id": "877104f7-e885-42b9-8de8-f6e4c6303969", + "model_name": "magpie-tts-multilingual", + }, + params: Optional[InputParams] = None, + **kwargs, + ): + """Initialize the NVIDIA Riva TTS service. + + Args: + api_key: NVIDIA API key for authentication. + server: gRPC server endpoint. Defaults to NVIDIA's cloud endpoint. + voice_id: Voice model identifier. Defaults to multilingual Ray voice. + sample_rate: Audio sample rate. If None, uses service default. + model_function_map: Dictionary containing function_id and model_name for the TTS model. + params: Additional configuration parameters for TTS synthesis. + **kwargs: Additional arguments passed to parent TTSService. + """ + super().__init__(sample_rate=sample_rate, **kwargs) + + params = params or NvidiaTTSService.InputParams() + + self._api_key = api_key + self._voice_id = voice_id + self._language_code = params.language + self._quality = params.quality + self._function_id = model_function_map.get("function_id") + + self.set_model_name(model_function_map.get("model_name")) + self.set_voice(voice_id) + + metadata = [ + ["function-id", self._function_id], + ["authorization", f"Bearer {api_key}"], + ] + auth = riva.client.Auth(None, True, server, metadata) + + self._service = riva.client.SpeechSynthesisService(auth) + + # warm up the service + config_response = self._service.stub.GetRivaSynthesisConfig( + riva.client.proto.riva_tts_pb2.RivaSynthesisConfigRequest() + ) + + async def set_model(self, model: str): + """Attempt to set the TTS model. + + Note: Model cannot be changed after initialization for Riva service. + + Args: + model: The model name to set (operation not supported). + """ + logger.warning(f"Cannot set model after initialization. Set model and function id like so:") + example = {"function_id": "", "model_name": ""} + logger.warning( + f"{self.__class__.__name__}(api_key=, model_function_map={example})" + ) + + @traced_tts + async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using NVIDIA Riva TTS. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech data. + """ + + def read_audio_responses(queue: asyncio.Queue): + def add_response(r): + asyncio.run_coroutine_threadsafe(queue.put(r), self.get_event_loop()) + + try: + responses = self._service.synthesize_online( + text, + self._voice_id, + self._language_code, + sample_rate_hz=self.sample_rate, + zero_shot_audio_prompt_file=None, + zero_shot_quality=self._quality, + custom_dictionary={}, + ) + for r in responses: + add_response(r) + add_response(None) + except Exception as e: + logger.error(f"{self} exception: {e}") + add_response(None) + + await self.start_ttfb_metrics() + yield TTSStartedFrame() + + logger.debug(f"{self}: Generating TTS [{text}]") + + try: + queue = asyncio.Queue() + await asyncio.to_thread(read_audio_responses, queue) + + # Wait for the thread to start. + resp = await asyncio.wait_for(queue.get(), timeout=NVIDIA_TTS_TIMEOUT_SECS) + while resp: + await self.stop_ttfb_metrics() + frame = TTSAudioRawFrame( + audio=resp.audio, + sample_rate=self.sample_rate, + num_channels=1, + ) + yield frame + resp = await asyncio.wait_for(queue.get(), timeout=NVIDIA_TTS_TIMEOUT_SECS) + except asyncio.TimeoutError: + logger.error(f"{self} timeout waiting for audio response") + yield ErrorFrame(error=f"{self} error: {e}") + + await self.start_tts_usage_metrics(text) + yield TTSStoppedFrame() diff --git a/src/pipecat/services/openai/base_llm.py b/src/pipecat/services/openai/base_llm.py index d020e1106..3fa9ae216 100644 --- a/src/pipecat/services/openai/base_llm.py +++ b/src/pipecat/services/openai/base_llm.py @@ -133,6 +133,7 @@ class BaseOpenAILLMService(LLMService): self._retry_timeout_secs = retry_timeout_secs self._retry_on_timeout = retry_on_timeout self.set_model_name(model) + self._full_model_name: str = "" self._client = self.create_client( api_key=api_key, base_url=base_url, @@ -185,6 +186,22 @@ class BaseOpenAILLMService(LLMService): """ return True + def set_full_model_name(self, full_model_name: str): + """Set the full AI model name. + + Args: + full_model_name: The full name of the AI model to use. + """ + self._full_model_name = full_model_name + + def get_full_model_name(self): + """Get the current full model name. + + Returns: + The full name of the AI model being used. + """ + return self._full_model_name + async def get_chat_completions( self, params_from_context: OpenAILLMInvocationParams ) -> AsyncStream[ChatCompletionChunk]: @@ -346,14 +363,23 @@ class BaseOpenAILLMService(LLMService): if chunk.usage.prompt_tokens_details else None ) + reasoning_tokens = ( + chunk.usage.completion_tokens_details.reasoning_tokens + if chunk.usage.completion_tokens_details + else None + ) tokens = LLMTokenUsage( prompt_tokens=chunk.usage.prompt_tokens, completion_tokens=chunk.usage.completion_tokens, total_tokens=chunk.usage.total_tokens, cache_read_input_tokens=cached_tokens, + reasoning_tokens=reasoning_tokens, ) await self.start_llm_usage_metrics(tokens) + if chunk.model and self.get_full_model_name() != chunk.model: + self.set_full_model_name(chunk.model) + if chunk.choices is None or len(chunk.choices) == 0: continue diff --git a/src/pipecat/services/openai/image.py b/src/pipecat/services/openai/image.py index 9501aace9..9daa8f2b9 100644 --- a/src/pipecat/services/openai/image.py +++ b/src/pipecat/services/openai/image.py @@ -76,7 +76,6 @@ class OpenAIImageGenService(ImageGenService): image_url = image.data[0].url if not image_url: - logger.error(f"{self} No image provided in response: {image}") yield ErrorFrame("Image generation failed") return diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index 8eaa3d6fa..7a57bf88c 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -19,6 +19,7 @@ from pipecat.adapters.services.open_ai_realtime_adapter import ( OpenAIRealtimeLLMAdapter, ) from pipecat.frames.frames import ( + AggregationType, BotStoppedSpeakingFrame, CancelFrame, EndFrame, @@ -56,7 +57,6 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import FunctionCallFromLLM, LLMService -from pipecat.services.openai.llm import OpenAIContextAggregatorPair from pipecat.transcriptions.language import Language from pipecat.utils.time import time_now_iso8601 from pipecat.utils.tracing.service_decorators import traced_openai_realtime, traced_stt @@ -443,7 +443,7 @@ class OpenAIRealtimeLLMService(LLMService): ) self._receive_task = self.create_task(self._receive_task_handler()) except Exception as e: - logger.error(f"{self} initialization error: {e}") + await self.push_error(error_msg=f"Error connecting: {e}", exception=e) self._websocket = None async def _disconnect(self): @@ -460,7 +460,7 @@ class OpenAIRealtimeLLMService(LLMService): self._completed_tool_calls = set() self._disconnecting = False except Exception as e: - logger.error(f"{self} error disconnecting: {e}") + await self.push_error(error_msg=f"Error disconnecting: {e}", exception=e) async def _ws_send(self, realtime_message): try: @@ -473,12 +473,11 @@ class OpenAIRealtimeLLMService(LLMService): # somehow *started* the websocket send attempt while we still # had a connection) return - logger.error(f"Error sending message to websocket: {e}") # In server-to-server contexts, a WebSocket error should be quite rare. Given how hard # it is to recover from a send-side error with proper state management, and that exponential # backoff for retries can have cost/stability implications for a service cluster, let's just # treat a send-side error as fatal. - await self.push_error(ErrorFrame(error=f"Error sending client event: {e}")) + await self.push_error(error_msg=f"Error sending client event: {e}", exception=e) async def _update_settings(self): settings = self._session_properties @@ -656,10 +655,17 @@ class OpenAIRealtimeLLMService(LLMService): async def _handle_evt_response_done(self, evt): # todo: figure out whether there's anything we need to do for "cancelled" events # usage metrics + cached_tokens = ( + evt.response.usage.input_token_details.cached_tokens + if hasattr(evt.response.usage, "input_token_details") + and evt.response.usage.input_token_details + else None + ) tokens = LLMTokenUsage( prompt_tokens=evt.response.usage.input_tokens, completion_tokens=evt.response.usage.output_tokens, total_tokens=evt.response.usage.total_tokens, + cache_read_input_tokens=cached_tokens, ) await self.start_llm_usage_metrics(tokens) await self.stop_processing_metrics() @@ -667,7 +673,7 @@ class OpenAIRealtimeLLMService(LLMService): self._current_assistant_response = None # error handling if evt.response.status == "failed": - await self.push_error(ErrorFrame(error=evt.response.status_details["error"]["message"])) + await self.push_error(error_msg=evt.response.status_details["error"]["message"]) return # response content for item in evt.response.output: @@ -684,7 +690,7 @@ class OpenAIRealtimeLLMService(LLMService): # We receive audio transcript deltas (as opposed to text deltas) when # the output modality is "audio" (the default) if evt.delta: - frame = TTSTextFrame(evt.delta) + frame = TTSTextFrame(evt.delta, aggregated_by=AggregationType.SENTENCE) # OpenAI Realtime text already includes any necessary inter-chunk spaces frame.includes_inter_frame_spaces = True await self.push_frame(frame) @@ -759,7 +765,7 @@ class OpenAIRealtimeLLMService(LLMService): async def _handle_evt_error(self, evt): # Errors are fatal to this connection. Send an ErrorFrame. - await self.push_error(ErrorFrame(error=f"Error: {evt}")) + await self.push_error(error_msg=f"Error: {evt}") # # state and client events for the current conversation @@ -809,7 +815,7 @@ class OpenAIRealtimeLLMService(LLMService): # We're done configuring the LLM for this session self._llm_needs_conversation_setup = False - logger.debug(f"Creating response") + logger.debug("Creating response") await self.push_frame(LLMFullResponseStartFrame()) await self.start_processing_metrics() diff --git a/src/pipecat/services/openai/tts.py b/src/pipecat/services/openai/tts.py index 23cb75324..aab1c8313 100644 --- a/src/pipecat/services/openai/tts.py +++ b/src/pipecat/services/openai/tts.py @@ -206,5 +206,4 @@ class OpenAITTSService(TTSService): yield frame yield TTSStoppedFrame() except BadRequestError as e: - logger.exception(f"{self} error generating TTS: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") diff --git a/src/pipecat/services/openai_realtime_beta/azure.py b/src/pipecat/services/openai_realtime_beta/azure.py index 2ec556ee3..e55dd075f 100644 --- a/src/pipecat/services/openai_realtime_beta/azure.py +++ b/src/pipecat/services/openai_realtime_beta/azure.py @@ -79,5 +79,5 @@ class AzureRealtimeBetaLLMService(OpenAIRealtimeBetaLLMService): ) self._receive_task = self.create_task(self._receive_task_handler()) except Exception as e: - logger.error(f"{self} initialization error: {e}") + await self.push_error(error_msg=f"Error connecting: {e}", exception=e) self._websocket = None diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index af0600882..16a00d08f 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -17,6 +17,7 @@ from loguru import logger from pipecat.adapters.services.open_ai_realtime_adapter import OpenAIRealtimeLLMAdapter from pipecat.frames.frames import ( + AggregationType, BotStoppedSpeakingFrame, CancelFrame, EndFrame, @@ -424,7 +425,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): ) self._receive_task = self.create_task(self._receive_task_handler()) except Exception as e: - logger.error(f"{self} initialization error: {e}") + await self.push_error(error_msg=f"Error connecting: {e}", exception=e) self._websocket = None async def _disconnect(self): @@ -440,7 +441,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): self._receive_task = None self._disconnecting = False except Exception as e: - logger.error(f"{self} error disconnecting: {e}") + await self.push_error(error_msg=f"Error disconnecting: {e}", exception=e) async def _ws_send(self, realtime_message): try: @@ -449,12 +450,11 @@ class OpenAIRealtimeBetaLLMService(LLMService): except Exception as e: if self._disconnecting: return - logger.error(f"Error sending message to websocket: {e}") # In server-to-server contexts, a WebSocket error should be quite rare. Given how hard # it is to recover from a send-side error with proper state management, and that exponential # backoff for retries can have cost/stability implications for a service cluster, let's just # treat a send-side error as fatal. - await self.push_error(ErrorFrame(error=f"Error sending client event: {e}")) + await self.push_error(error_msg=f"Error sending client event: {e}", exception=e) async def _update_settings(self): settings = self._session_properties @@ -652,7 +652,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): async def _handle_evt_audio_transcript_delta(self, evt): if evt.delta: await self.push_frame(LLMTextFrame(evt.delta)) - await self.push_frame(TTSTextFrame(evt.delta)) + await self.push_frame(TTSTextFrame(evt.delta, aggregated_by=AggregationType.SENTENCE)) async def _handle_evt_speech_started(self, evt): await self._truncate_current_audio_response() @@ -685,7 +685,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): async def _handle_evt_error(self, evt): # Errors are fatal to this connection. Send an ErrorFrame. - await self.push_error(ErrorFrame(error=f"Error: {evt}")) + await self.push_error(error_msg=f"Error: {evt}") async def _handle_assistant_output(self, output): # We haven't seen intermixed audio and function_call items in the same response. But let's diff --git a/src/pipecat/services/piper/tts.py b/src/pipecat/services/piper/tts.py index dd842ff11..6bd9374bc 100644 --- a/src/pipecat/services/piper/tts.py +++ b/src/pipecat/services/piper/tts.py @@ -88,9 +88,6 @@ class PiperTTSService(TTSService): ) as response: if response.status != 200: error = await response.text() - logger.error( - f"{self} error getting audio (status: {response.status}, error: {error})" - ) yield ErrorFrame( error=f"Error getting audio (status: {response.status}, error: {error})" ) @@ -109,7 +106,7 @@ class PiperTTSService(TTSService): yield frame except Exception as e: logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") finally: logger.debug(f"{self}: Finished TTS [{text}]") await self.stop_ttfb_metrics() diff --git a/src/pipecat/services/playht/tts.py b/src/pipecat/services/playht/tts.py index 2c40065aa..5af54a9aa 100644 --- a/src/pipecat/services/playht/tts.py +++ b/src/pipecat/services/playht/tts.py @@ -266,8 +266,7 @@ class PlayHTTTSService(InterruptibleTTSService): self._websocket = None await self._call_event_handler("on_connection_error", f"{e}") except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Error connecting: {e}", exception=e) self._websocket = None await self._call_event_handler("on_connection_error", f"{e}") @@ -280,8 +279,7 @@ class PlayHTTTSService(InterruptibleTTSService): logger.debug("Disconnecting from PlayHT") await self._websocket.close() except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Error disconnecting: {e}", exception=e) finally: self._request_id = None self._websocket = None @@ -351,8 +349,7 @@ class PlayHTTTSService(InterruptibleTTSService): await self.push_frame(TTSStoppedFrame()) self._request_id = None elif "error" in msg: - logger.error(f"{self} error: {msg}") - await self.push_error(ErrorFrame(error=f"{self} error: {msg['error']}")) + await self.push_error(error_msg=f"Error: {msg['error']}") except json.JSONDecodeError: logger.error(f"Invalid JSON message: {message}") @@ -394,8 +391,7 @@ class PlayHTTTSService(InterruptibleTTSService): await self._get_websocket().send(json.dumps(tts_command)) await self.start_tts_usage_metrics(text) except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") yield TTSStoppedFrame() await self._disconnect() await self._connect() @@ -405,8 +401,7 @@ class PlayHTTTSService(InterruptibleTTSService): yield None except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") class PlayHTHttpTTSService(TTSService): @@ -626,8 +621,7 @@ class PlayHTHttpTTSService(TTSService): yield frame except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") finally: await self.stop_ttfb_metrics() yield TTSStoppedFrame() diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index d424491b9..df71668b7 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -118,6 +118,10 @@ class RimeTTSService(AudioContextWordTTSService): sample_rate: Audio sample rate in Hz. params: Additional configuration parameters. text_aggregator: Custom text aggregator for processing input text. + + .. deprecated:: 0.0.95 + Use an LLMTextProcessor before the TTSService for custom text aggregation. + aggregate_sentences: Whether to aggregate sentences within the TTSService. **kwargs: Additional arguments passed to parent class. """ @@ -128,10 +132,17 @@ class RimeTTSService(AudioContextWordTTSService): push_stop_frames=True, pause_frame_processing=True, sample_rate=sample_rate, - text_aggregator=text_aggregator or SkipTagsAggregator([("spell(", ")")]), **kwargs, ) + if not text_aggregator: + # Always skip tags added for spelled-out text + # Note: This is primarily to support backwards compatibility. + # The preferred way of taking advantage of Rime spelling is + # to use an LLMTextProcessor and/or a text_transformer to identify + # and insert these tags for the purpose of the TTS service alone. + self._text_aggregator = SkipTagsAggregator([("spell(", ")")]) + params = params or RimeTTSService.InputParams() # Store service configuration @@ -157,6 +168,7 @@ class RimeTTSService(AudioContextWordTTSService): self._context_id = None # Tracks current turn self._receive_task = None self._cumulative_time = 0 # Accumulates time across messages + self._extra_msg_fields = {} # Extra fields for next message def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -186,6 +198,31 @@ class RimeTTSService(AudioContextWordTTSService): self._model = model await super().set_model(model) + # A set of Rime-specific helpers for text transformations + def SPELL(text: str) -> str: + """Wrap text in Rime spell function.""" + return f"spell({text})" + + def PAUSE_TAG(seconds: float) -> str: + """Convenience method to create a pause tag.""" + return f"<{seconds * 1000}>" + + def PRONOUNCE(self, text: str, word: str, phoneme: str) -> str: + """Convenience method to support Rime's custom pronunciations feature. + + https://docs.rime.ai/api-reference/custom-pronunciation + """ + self._extra_msg_fields["phonemizeBetweenBrackets"] = True + return text.replace(word, f"{phoneme}") + + def INLINE_SPEED(self, text: str, speed: float) -> str: + """Convenience method to support inline speeds.""" + if not self._extra_msg_fields: + self._extra_msg_fields = {} + speed_vals = self._extra_msg_fields.get("inlineSpeedAlpha", "").split(",") + self._extra_msg_fields["inlineSpeedAlpha"] = ",".join(speed_vals + [str(speed)]) + return f"[{text}]" + async def _update_settings(self, settings: Mapping[str, Any]): """Update service settings and reconnect if voice changed.""" prev_voice = self._voice_id @@ -198,7 +235,11 @@ class RimeTTSService(AudioContextWordTTSService): def _build_msg(self, text: str = "") -> dict: """Build JSON message for Rime API.""" - return {"text": text, "contextId": self._context_id} + msg = {"text": text, "contextId": self._context_id} + if self._extra_msg_fields: + msg |= self._extra_msg_fields + self._extra_msg_fields = {} + return msg def _build_clear_msg(self) -> dict: """Build clear operation message.""" @@ -264,8 +305,7 @@ class RimeTTSService(AudioContextWordTTSService): await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Error connecting: {e}", exception=e) self._websocket = None await self._call_event_handler("on_connection_error", f"{e}") @@ -277,8 +317,7 @@ class RimeTTSService(AudioContextWordTTSService): await self._websocket.send(json.dumps(self._build_eos_msg())) await self._websocket.close() except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Error disconnecting: {e}", exception=e) finally: self._context_id = None self._websocket = None @@ -371,10 +410,9 @@ class RimeTTSService(AudioContextWordTTSService): logger.debug(f"Updated cumulative time to: {self._cumulative_time}") elif msg["type"] == "error": - logger.error(f"{self} error: {msg}") await self.push_frame(TTSStoppedFrame()) await self.stop_all_metrics() - await self.push_error(ErrorFrame(error=f"{self} error: {msg['message']}")) + await self.push_error(error_msg=f"Error: {msg['message']}") self._context_id = None async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): @@ -416,16 +454,14 @@ class RimeTTSService(AudioContextWordTTSService): await self._get_websocket().send(json.dumps(msg)) await self.start_tts_usage_metrics(text) except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") yield TTSStoppedFrame() await self._disconnect() await self._connect() return yield None except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") class RimeHttpTTSService(TTSService): @@ -556,7 +592,6 @@ class RimeHttpTTSService(TTSService): ) as response: if response.status != 200: error_message = f"Rime TTS error: HTTP {response.status}" - logger.error(error_message) yield ErrorFrame(error=error_message) return @@ -574,8 +609,7 @@ class RimeHttpTTSService(TTSService): yield frame except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") finally: await self.stop_ttfb_metrics() yield TTSStoppedFrame() diff --git a/src/pipecat/services/riva/stt.py b/src/pipecat/services/riva/stt.py index 25cc78c7a..9dcb44948 100644 --- a/src/pipecat/services/riva/stt.py +++ b/src/pipecat/services/riva/stt.py @@ -4,709 +4,32 @@ # SPDX-License-Identifier: BSD 2-Clause License # -"""NVIDIA Riva Speech-to-Text service implementations for real-time and batch transcription.""" +"""NVIDIA Riva Speech-to-Text service implementations for real-time and batch transcription. -import asyncio -from concurrent.futures import CancelledError as FuturesCancelledError -from typing import AsyncGenerator, List, Mapping, Optional +.. deprecated:: 0.0.96 + This module is deprecated. Please NvidiaSTTService from + pipecat.services.nvidia.stt instead. +""" -from loguru import logger -from pydantic import BaseModel +import warnings -from pipecat.frames.frames import ( - CancelFrame, - EndFrame, - ErrorFrame, - Frame, - InterimTranscriptionFrame, - StartFrame, - TranscriptionFrame, +from pipecat.services.nvidia.stt import ( + NvidiaSegmentedSTTService, + NvidiaSTTService, + language_to_nvidia_riva_language, ) -from pipecat.services.stt_service import SegmentedSTTService, STTService -from pipecat.transcriptions.language import Language, resolve_language -from pipecat.utils.time import time_now_iso8601 -from pipecat.utils.tracing.service_decorators import traced_stt -try: - import riva.client - -except ModuleNotFoundError as e: - logger.error(f"Exception: {e}") - logger.error("In order to use NVIDIA Riva STT, you need to `pip install pipecat-ai[riva]`.") - raise Exception(f"Missing module: {e}") - - -def language_to_riva_language(language: Language) -> Optional[str]: - """Maps Language enum to Riva ASR language codes. - - Source: - https://docs.nvidia.com/deeplearning/riva/user-guide/docs/asr/asr-riva-build-table.html?highlight=fr%20fr - - Args: - language: Language enum value. - - Returns: - Optional[str]: Riva language code or None if not supported. - """ - LANGUAGE_MAP = { - # Arabic - Language.AR: "ar-AR", - # English - Language.EN: "en-US", # Default to US - Language.EN_US: "en-US", - Language.EN_GB: "en-GB", - # French - Language.FR: "fr-FR", - Language.FR_FR: "fr-FR", - # German - Language.DE: "de-DE", - Language.DE_DE: "de-DE", - # Hindi - Language.HI: "hi-IN", - Language.HI_IN: "hi-IN", - # Italian - Language.IT: "it-IT", - Language.IT_IT: "it-IT", - # Japanese - Language.JA: "ja-JP", - Language.JA_JP: "ja-JP", - # Korean - Language.KO: "ko-KR", - Language.KO_KR: "ko-KR", - # Portuguese - Language.PT: "pt-BR", # Default to Brazilian - Language.PT_BR: "pt-BR", - # Russian - Language.RU: "ru-RU", - Language.RU_RU: "ru-RU", - # Spanish - Language.ES: "es-ES", # Default to Spain - Language.ES_ES: "es-ES", - Language.ES_US: "es-US", # US Spanish - } - - return resolve_language(language, LANGUAGE_MAP, use_base_code=False) - - -class RivaSTTService(STTService): - """Real-time speech-to-text service using NVIDIA Riva streaming ASR. - - Provides real-time transcription capabilities using NVIDIA's Riva ASR models - through streaming recognition. Supports interim results and continuous audio - processing for low-latency applications. - """ - - class InputParams(BaseModel): - """Configuration parameters for Riva STT service. - - Parameters: - language: Target language for transcription. Defaults to EN_US. - """ - - language: Optional[Language] = Language.EN_US - - def __init__( - self, - *, - api_key: str, - server: str = "grpc.nvcf.nvidia.com:443", - model_function_map: Mapping[str, str] = { - "function_id": "1598d209-5e27-4d3c-8079-4751568b1081", - "model_name": "parakeet-ctc-1.1b-asr", - }, - sample_rate: Optional[int] = None, - params: Optional[InputParams] = None, - **kwargs, - ): - """Initialize the Riva STT service. - - Args: - api_key: NVIDIA API key for authentication. - server: Riva server address. Defaults to NVIDIA Cloud Function endpoint. - model_function_map: Mapping containing 'function_id' and 'model_name' for the ASR model. - sample_rate: Audio sample rate in Hz. If None, uses pipeline default. - params: Additional configuration parameters for Riva. - **kwargs: Additional arguments passed to STTService. - """ - super().__init__(sample_rate=sample_rate, **kwargs) - - params = params or RivaSTTService.InputParams() - - self._api_key = api_key - self._profanity_filter = False - self._automatic_punctuation = True - self._no_verbatim_transcripts = False - self._language_code = params.language - self._boosted_lm_words = None - self._boosted_lm_score = 4.0 - self._start_history = -1 - self._start_threshold = -1.0 - self._stop_history = -1 - self._stop_threshold = -1.0 - self._stop_history_eou = -1 - self._stop_threshold_eou = -1.0 - self._custom_configuration = "" - self._function_id = model_function_map.get("function_id") - - self._settings = { - "language": str(params.language), - "profanity_filter": self._profanity_filter, - "automatic_punctuation": self._automatic_punctuation, - "verbatim_transcripts": not self._no_verbatim_transcripts, - "boosted_lm_words": self._boosted_lm_words, - "boosted_lm_score": self._boosted_lm_score, - } - - self.set_model_name(model_function_map.get("model_name")) - - metadata = [ - ["function-id", self._function_id], - ["authorization", f"Bearer {api_key}"], - ] - auth = riva.client.Auth(None, True, server, metadata) - - self._asr_service = riva.client.ASRService(auth) - - self._queue = None - self._config = None - self._thread_task = None - self._response_task = None - - def can_generate_metrics(self) -> bool: - """Check if this service can generate processing metrics. - - Returns: - False - this service does not support metrics generation. - """ - return False - - async def set_model(self, model: str): - """Set the ASR model for transcription. - - Args: - model: Model name to set. - - Note: - Model cannot be changed after initialization. Use model_function_map - parameter in constructor instead. - """ - logger.warning(f"Cannot set model after initialization. Set model and function id like so:") - example = {"function_id": "", "model_name": ""} - logger.warning( - f"{self.__class__.__name__}(api_key=, model_function_map={example})" - ) - - async def start(self, frame: StartFrame): - """Start the Riva STT service and initialize streaming configuration. - - Args: - frame: StartFrame indicating pipeline start. - """ - await super().start(frame) - - if self._config: - return - - config = riva.client.StreamingRecognitionConfig( - config=riva.client.RecognitionConfig( - encoding=riva.client.AudioEncoding.LINEAR_PCM, - language_code=self._language_code, - model="", - max_alternatives=1, - profanity_filter=self._profanity_filter, - enable_automatic_punctuation=self._automatic_punctuation, - verbatim_transcripts=not self._no_verbatim_transcripts, - sample_rate_hertz=self.sample_rate, - audio_channel_count=1, - ), - interim_results=True, - ) - - riva.client.add_word_boosting_to_config( - config, self._boosted_lm_words, self._boosted_lm_score - ) - - riva.client.add_endpoint_parameters_to_config( - config, - self._start_history, - self._start_threshold, - self._stop_history, - self._stop_history_eou, - self._stop_threshold, - self._stop_threshold_eou, - ) - riva.client.add_custom_configuration_to_config(config, self._custom_configuration) - - self._config = config - self._queue = asyncio.Queue() - - if not self._thread_task: - self._thread_task = self.create_task(self._thread_task_handler()) - - if not self._response_task: - self._response_queue = asyncio.Queue() - self._response_task = self.create_task(self._response_task_handler()) - - async def stop(self, frame: EndFrame): - """Stop the Riva STT service and clean up resources. - - Args: - frame: EndFrame indicating pipeline stop. - """ - await super().stop(frame) - await self._stop_tasks() - - async def cancel(self, frame: CancelFrame): - """Cancel the Riva STT service operation. - - Args: - frame: CancelFrame indicating operation cancellation. - """ - await super().cancel(frame) - await self._stop_tasks() - - async def _stop_tasks(self): - if self._thread_task: - await self.cancel_task(self._thread_task) - self._thread_task = None - - if self._response_task: - await self.cancel_task(self._response_task) - self._response_task = None - - def _response_handler(self): - responses = self._asr_service.streaming_response_generator( - audio_chunks=self, - streaming_config=self._config, - ) - for response in responses: - if not response.results: - continue - asyncio.run_coroutine_threadsafe( - self._response_queue.put(response), self.get_event_loop() - ) - - async def _thread_task_handler(self): - try: - self._thread_running = True - await asyncio.to_thread(self._response_handler) - except asyncio.CancelledError: - self._thread_running = False - raise - - @traced_stt - async def _handle_transcription( - self, transcript: str, is_final: bool, language: Optional[Language] = None - ): - """Handle a transcription result with tracing.""" - pass - - async def _handle_response(self, response): - for result in response.results: - if result and not result.alternatives: - continue - - transcript = result.alternatives[0].transcript - if transcript and len(transcript) > 0: - await self.stop_ttfb_metrics() - if result.is_final: - await self.stop_processing_metrics() - await self.push_frame( - TranscriptionFrame( - transcript, - self._user_id, - time_now_iso8601(), - self._language_code, - result=result, - ) - ) - await self._handle_transcription( - transcript=transcript, - is_final=result.is_final, - language=self._language_code, - ) - else: - await self.push_frame( - InterimTranscriptionFrame( - transcript, - self._user_id, - time_now_iso8601(), - self._language_code, - result=result, - ) - ) - - async def _response_task_handler(self): - while True: - response = await self._response_queue.get() - await self._handle_response(response) - self._response_queue.task_done() - - async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: - """Process audio data for speech-to-text transcription. - - Args: - audio: Raw audio bytes to transcribe. - - Yields: - None - transcription results are pushed to the pipeline via frames. - """ - await self.start_ttfb_metrics() - await self.start_processing_metrics() - await self._queue.put(audio) - yield None - - def __next__(self) -> bytes: - """Get the next audio chunk for Riva processing. - - Returns: - Audio bytes from the queue. - - Raises: - StopIteration: When the thread is no longer running. - """ - if not self._thread_running: - raise StopIteration - - try: - future = asyncio.run_coroutine_threadsafe(self._queue.get(), self.get_event_loop()) - return future.result() - except FuturesCancelledError: - raise StopIteration - - def __iter__(self): - """Return iterator for audio chunk processing. - - Returns: - Self as iterator. - """ - return self - - -class RivaSegmentedSTTService(SegmentedSTTService): - """Speech-to-text service using NVIDIA Riva's offline/batch models. - - By default, his service uses NVIDIA's Riva Canary ASR API to perform speech-to-text - transcription on audio segments. It inherits from SegmentedSTTService to handle - audio buffering and speech detection. - """ - - class InputParams(BaseModel): - """Configuration parameters for Riva segmented STT service. - - Parameters: - language: Target language for transcription. Defaults to EN_US. - profanity_filter: Whether to filter profanity from results. - automatic_punctuation: Whether to add automatic punctuation. - verbatim_transcripts: Whether to return verbatim transcripts. - boosted_lm_words: List of words to boost in language model. - boosted_lm_score: Score boost for specified words. - """ - - language: Optional[Language] = Language.EN_US - profanity_filter: bool = False - automatic_punctuation: bool = True - verbatim_transcripts: bool = False - boosted_lm_words: Optional[List[str]] = None - boosted_lm_score: float = 4.0 - - def __init__( - self, - *, - api_key: str, - server: str = "grpc.nvcf.nvidia.com:443", - model_function_map: Mapping[str, str] = { - "function_id": "ee8dc628-76de-4acc-8595-1836e7e857bd", - "model_name": "canary-1b-asr", - }, - sample_rate: Optional[int] = None, - params: Optional[InputParams] = None, - **kwargs, - ): - """Initialize the Riva segmented STT service. - - Args: - api_key: NVIDIA API key for authentication - server: Riva server address (defaults to NVIDIA Cloud Function endpoint) - model_function_map: Mapping of model name and its corresponding NVIDIA Cloud Function ID - sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate - params: Additional configuration parameters for Riva - **kwargs: Additional arguments passed to SegmentedSTTService - """ - super().__init__(sample_rate=sample_rate, **kwargs) - - params = params or RivaSegmentedSTTService.InputParams() - - # Set model name - self.set_model_name(model_function_map.get("model_name")) - - # Initialize Riva settings - self._api_key = api_key - self._server = server - self._function_id = model_function_map.get("function_id") - self._model_name = model_function_map.get("model_name") - - # Store the language as a Language enum and as a string - self._language_enum = params.language or Language.EN_US - self._language = self.language_to_service_language(self._language_enum) or "en-US" - - # Configure transcription parameters - self._profanity_filter = params.profanity_filter - self._automatic_punctuation = params.automatic_punctuation - self._verbatim_transcripts = params.verbatim_transcripts - self._boosted_lm_words = params.boosted_lm_words - self._boosted_lm_score = params.boosted_lm_score - - # Voice activity detection thresholds (use Riva defaults) - self._start_history = -1 - self._start_threshold = -1.0 - self._stop_history = -1 - self._stop_threshold = -1.0 - self._stop_history_eou = -1 - self._stop_threshold_eou = -1.0 - self._custom_configuration = "" - - # Create Riva client - self._config = None - self._asr_service = None - self._settings = {"language": self._language_enum} - - def language_to_service_language(self, language: Language) -> Optional[str]: - """Convert pipecat Language enum to Riva's language code. - - Args: - language: Language enum value. - - Returns: - Riva language code or None if not supported. - """ - return language_to_riva_language(language) - - def _initialize_client(self): - """Initialize the Riva ASR client with authentication metadata.""" - if self._asr_service is not None: - return - - # Set up authentication metadata for NVIDIA Cloud Functions - metadata = [ - ["function-id", self._function_id], - ["authorization", f"Bearer {self._api_key}"], - ] - - # Create authenticated client - auth = riva.client.Auth(None, True, self._server, metadata) - self._asr_service = riva.client.ASRService(auth) - - logger.info(f"Initialized RivaSegmentedSTTService with model: {self.model_name}") - - def _create_recognition_config(self): - """Create the Riva ASR recognition configuration.""" - # Create base configuration - config = riva.client.RecognitionConfig( - language_code=self._language, # Now using the string, not a tuple - max_alternatives=1, - profanity_filter=self._profanity_filter, - enable_automatic_punctuation=self._automatic_punctuation, - verbatim_transcripts=self._verbatim_transcripts, - ) - - # Add word boosting if specified - if self._boosted_lm_words: - riva.client.add_word_boosting_to_config( - config, self._boosted_lm_words, self._boosted_lm_score - ) - - # Add voice activity detection parameters - riva.client.add_endpoint_parameters_to_config( - config, - self._start_history, - self._start_threshold, - self._stop_history, - self._stop_history_eou, - self._stop_threshold, - self._stop_threshold_eou, - ) - - # Add any custom configuration - if self._custom_configuration: - riva.client.add_custom_configuration_to_config(config, self._custom_configuration) - - return config - - def can_generate_metrics(self) -> bool: - """Check if this service can generate processing metrics. - - Returns: - True - this service supports metrics generation. - """ - return True - - async def set_model(self, model: str): - """Set the ASR model for transcription. - - Args: - model: Model name to set. - - Note: - Model cannot be changed after initialization. Use model_function_map - parameter in constructor instead. - """ - logger.warning(f"Cannot set model after initialization. Set model and function id like so:") - example = {"function_id": "", "model_name": ""} - logger.warning( - f"{self.__class__.__name__}(api_key=, model_function_map={example})" - ) - - async def start(self, frame: StartFrame): - """Initialize the service when the pipeline starts. - - Args: - frame: StartFrame indicating pipeline start. - """ - await super().start(frame) - self._initialize_client() - self._config = self._create_recognition_config() - - async def set_language(self, language: Language): - """Set the language for the STT service. - - Args: - language: Target language for transcription. - """ - logger.info(f"Switching STT language to: [{language}]") - self._language_enum = language - self._language = self.language_to_service_language(language) or "en-US" - self._settings["language"] = language - - # Update configuration with new language - if self._config: - self._config.language_code = self._language - - @traced_stt - async def _handle_transcription( - self, transcript: str, is_final: bool, language: Optional[Language] = None - ): - """Handle a transcription result with tracing.""" - pass - - async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: - """Transcribe an audio segment. - - Args: - audio: Raw audio bytes in WAV format (already converted by base class). - - Yields: - Frame: TranscriptionFrame containing the transcribed text. - """ - try: - await self.start_processing_metrics() - await self.start_ttfb_metrics() - - # Make sure the client is initialized - if self._asr_service is None: - self._initialize_client() - - # Make sure the config is created - if self._config is None: - self._config = self._create_recognition_config() - - # Type assertion to satisfy the IDE - assert self._asr_service is not None, "ASR service not initialized" - assert self._config is not None, "Recognition config not created" - - # Process audio with Riva ASR - explicitly request non-future response - raw_response = self._asr_service.offline_recognize(audio, self._config, future=False) - - await self.stop_ttfb_metrics() - await self.stop_processing_metrics() - - # Process the response - handle different possible return types - try: - # If it's a future-like object, get the result - if hasattr(raw_response, "result"): - response = raw_response.result() - else: - response = raw_response - - # Process transcription results - transcription_found = False - - # Now we can safely check results - # Type hint for the IDE - results = getattr(response, "results", []) - - for result in results: - alternatives = getattr(result, "alternatives", []) - if alternatives: - text = alternatives[0].transcript.strip() - if text: - logger.debug(f"Transcription: [{text}]") - yield TranscriptionFrame( - text, - self._user_id, - time_now_iso8601(), - self._language_enum, - ) - transcription_found = True - - await self._handle_transcription(text, True, self._language_enum) - - if not transcription_found: - logger.debug("No transcription results found in Riva response") - - except AttributeError as ae: - logger.error(f"Unexpected response structure from Riva: {ae}") - yield ErrorFrame(f"Unexpected Riva response format: {str(ae)}") - - except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") - - -class ParakeetSTTService(RivaSTTService): - """Deprecated speech-to-text service using NVIDIA Parakeet models. - - .. deprecated:: 0.0.66 - This class is deprecated. Use `RivaSTTService` instead for equivalent functionality - with Parakeet models by specifying the appropriate model_function_map. - """ - - def __init__( - self, - *, - api_key: str, - server: str = "grpc.nvcf.nvidia.com:443", - model_function_map: Mapping[str, str] = { - "function_id": "1598d209-5e27-4d3c-8079-4751568b1081", - "model_name": "parakeet-ctc-1.1b-asr", - }, - sample_rate: Optional[int] = None, - params: Optional[RivaSTTService.InputParams] = None, # Use parent class's type - **kwargs, - ): - """Initialize the Parakeet STT service. - - Args: - api_key: NVIDIA API key for authentication. - server: Riva server address. Defaults to NVIDIA Cloud Function endpoint. - model_function_map: Mapping containing 'function_id' and 'model_name' for Parakeet model. - sample_rate: Audio sample rate in Hz. If None, uses pipeline default. - params: Additional configuration parameters for Riva. - **kwargs: Additional arguments passed to RivaSTTService. - """ - super().__init__( - api_key=api_key, - server=server, - model_function_map=model_function_map, - sample_rate=sample_rate, - params=params, - **kwargs, - ) - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "`ParakeetSTTService` is deprecated, use `RivaSTTService` instead.", - DeprecationWarning, - ) +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "RivaSTTService and ParakeetSTTService " + "from pipecat.services.riva.stt is deprecated. " + "Please use NvidiaSTTService from pipecat.services.nvidia.stt instead.", + DeprecationWarning, + stacklevel=2, + ) + +RivaSTTService = NvidiaSTTService +language_to_riva_language = language_to_nvidia_riva_language +RivaSegmentedSTTService = NvidiaSegmentedSTTService +ParakeetSTTService = NvidiaSTTService diff --git a/src/pipecat/services/riva/tts.py b/src/pipecat/services/riva/tts.py index 8b9effbba..642409c8b 100644 --- a/src/pipecat/services/riva/tts.py +++ b/src/pipecat/services/riva/tts.py @@ -8,232 +8,26 @@ This module provides integration with NVIDIA Riva's TTS services through gRPC API for high-quality speech synthesis. + +.. deprecated:: 0.0.96 + This module is deprecated. Please NvidiaTTSService from + pipecat.services.nvidia.tts instead. """ -import asyncio -import os -from typing import AsyncGenerator, Mapping, Optional +import warnings -from pipecat.utils.tracing.service_decorators import traced_tts +from pipecat.services.nvidia.tts import NVIDIA_TTS_TIMEOUT_SECS, NvidiaTTSService -# Suppress gRPC fork warnings -os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false" +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "FastPitchTTSService and RivaTTSService " + "from pipecat.services.nim.llm are deprecated. " + "Please use NvidiaLLMService from pipecat.services.nvidia.tts instead.", + DeprecationWarning, + stacklevel=2, + ) -from loguru import logger -from pydantic import BaseModel - -from pipecat.frames.frames import ( - ErrorFrame, - Frame, - TTSAudioRawFrame, - TTSStartedFrame, - TTSStoppedFrame, -) -from pipecat.services.tts_service import TTSService -from pipecat.transcriptions.language import Language - -try: - import riva.client - -except ModuleNotFoundError as e: - logger.error(f"Exception: {e}") - logger.error("In order to use NVIDIA Riva TTS, you need to `pip install pipecat-ai[riva]`.") - raise Exception(f"Missing module: {e}") - -RIVA_TTS_TIMEOUT_SECS = 5 - - -class RivaTTSService(TTSService): - """NVIDIA Riva text-to-speech service. - - Provides high-quality text-to-speech synthesis using NVIDIA Riva's - cloud-based TTS models. Supports multiple voices, languages, and - configurable quality settings. - """ - - class InputParams(BaseModel): - """Input parameters for Riva TTS configuration. - - Parameters: - language: Language code for synthesis. Defaults to US English. - quality: Audio quality setting (0-100). Defaults to 20. - """ - - language: Optional[Language] = Language.EN_US - quality: Optional[int] = 20 - - def __init__( - self, - *, - api_key: str, - server: str = "grpc.nvcf.nvidia.com:443", - voice_id: str = "Magpie-Multilingual.EN-US.Aria", - sample_rate: Optional[int] = None, - model_function_map: Mapping[str, str] = { - "function_id": "877104f7-e885-42b9-8de8-f6e4c6303969", - "model_name": "magpie-tts-multilingual", - }, - params: Optional[InputParams] = None, - **kwargs, - ): - """Initialize the NVIDIA Riva TTS service. - - Args: - api_key: NVIDIA API key for authentication. - server: gRPC server endpoint. Defaults to NVIDIA's cloud endpoint. - voice_id: Voice model identifier. Defaults to multilingual Ray voice. - sample_rate: Audio sample rate. If None, uses service default. - model_function_map: Dictionary containing function_id and model_name for the TTS model. - params: Additional configuration parameters for TTS synthesis. - **kwargs: Additional arguments passed to parent TTSService. - """ - super().__init__(sample_rate=sample_rate, **kwargs) - - params = params or RivaTTSService.InputParams() - - self._api_key = api_key - self._voice_id = voice_id - self._language_code = params.language - self._quality = params.quality - self._function_id = model_function_map.get("function_id") - - self.set_model_name(model_function_map.get("model_name")) - self.set_voice(voice_id) - - metadata = [ - ["function-id", self._function_id], - ["authorization", f"Bearer {api_key}"], - ] - auth = riva.client.Auth(None, True, server, metadata) - - self._service = riva.client.SpeechSynthesisService(auth) - - # warm up the service - config_response = self._service.stub.GetRivaSynthesisConfig( - riva.client.proto.riva_tts_pb2.RivaSynthesisConfigRequest() - ) - - async def set_model(self, model: str): - """Attempt to set the TTS model. - - Note: Model cannot be changed after initialization for Riva service. - - Args: - model: The model name to set (operation not supported). - """ - logger.warning(f"Cannot set model after initialization. Set model and function id like so:") - example = {"function_id": "", "model_name": ""} - logger.warning( - f"{self.__class__.__name__}(api_key=, model_function_map={example})" - ) - - @traced_tts - async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - """Generate speech from text using NVIDIA Riva TTS. - - Args: - text: The text to synthesize into speech. - - Yields: - Frame: Audio frames containing the synthesized speech data. - """ - - def read_audio_responses(queue: asyncio.Queue): - def add_response(r): - asyncio.run_coroutine_threadsafe(queue.put(r), self.get_event_loop()) - - try: - responses = self._service.synthesize_online( - text, - self._voice_id, - self._language_code, - sample_rate_hz=self.sample_rate, - zero_shot_audio_prompt_file=None, - zero_shot_quality=self._quality, - custom_dictionary={}, - ) - for r in responses: - add_response(r) - add_response(None) - except Exception as e: - logger.error(f"{self} exception: {e}") - add_response(None) - - await self.start_ttfb_metrics() - yield TTSStartedFrame() - - logger.debug(f"{self}: Generating TTS [{text}]") - - try: - queue = asyncio.Queue() - await asyncio.to_thread(read_audio_responses, queue) - - # Wait for the thread to start. - resp = await asyncio.wait_for(queue.get(), timeout=RIVA_TTS_TIMEOUT_SECS) - while resp: - await self.stop_ttfb_metrics() - frame = TTSAudioRawFrame( - audio=resp.audio, - sample_rate=self.sample_rate, - num_channels=1, - ) - yield frame - resp = await asyncio.wait_for(queue.get(), timeout=RIVA_TTS_TIMEOUT_SECS) - except asyncio.TimeoutError: - logger.error(f"{self} timeout waiting for audio response") - yield ErrorFrame(error=f"{self} error: {e}") - - await self.start_tts_usage_metrics(text) - yield TTSStoppedFrame() - - -class FastPitchTTSService(RivaTTSService): - """Deprecated FastPitch TTS service. - - .. deprecated:: 0.0.66 - This class is deprecated. Use RivaTTSService instead for new implementations. - Provides backward compatibility for existing FastPitch TTS integrations. - """ - - def __init__( - self, - *, - api_key: str, - server: str = "grpc.nvcf.nvidia.com:443", - voice_id: str = "English-US.Female-1", - sample_rate: Optional[int] = None, - model_function_map: Mapping[str, str] = { - "function_id": "0149dedb-2be8-4195-b9a0-e57e0e14f972", - "model_name": "fastpitch-hifigan-tts", - }, - params: Optional[RivaTTSService.InputParams] = None, - **kwargs, - ): - """Initialize the deprecated FastPitch TTS service. - - Args: - api_key: NVIDIA API key for authentication. - server: gRPC server endpoint. Defaults to NVIDIA's cloud endpoint. - voice_id: Voice model identifier. Defaults to Female-1 voice. - sample_rate: Audio sample rate. If None, uses service default. - model_function_map: Dictionary containing function_id and model_name for FastPitch model. - params: Additional configuration parameters for TTS synthesis. - **kwargs: Additional arguments passed to parent RivaTTSService. - """ - super().__init__( - api_key=api_key, - server=server, - voice_id=voice_id, - sample_rate=sample_rate, - model_function_map=model_function_map, - params=params, - **kwargs, - ) - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "`FastPitchTTSService` is deprecated, use `RivaTTSService` instead.", - DeprecationWarning, - ) +RivaTTSService = NvidiaTTSService +FastPitchTTSService = NvidiaTTSService +RIVA_TTS_TIMEOUT_SECS = NVIDIA_TTS_TIMEOUT_SECS diff --git a/src/pipecat/services/sarvam/stt.py b/src/pipecat/services/sarvam/stt.py index 98e643fde..6f9a02642 100644 --- a/src/pipecat/services/sarvam/stt.py +++ b/src/pipecat/services/sarvam/stt.py @@ -275,8 +275,7 @@ class SarvamSTTService(STTService): await self._socket_client.translate(**method_kwargs) except Exception as e: - logger.error(f"Error sending audio to Sarvam: {e}") - await self.push_error(ErrorFrame(f"Failed to send audio: {e}")) + yield ErrorFrame(error=f"Error sending audio to Sarvam: {e}", exception=e) yield None @@ -332,13 +331,11 @@ class SarvamSTTService(STTService): logger.info("Connected to Sarvam successfully") except ApiError as e: - logger.error(f"Sarvam API error: {e}") - await self.push_error(ErrorFrame(f"Sarvam API error: {e}")) + await self.push_error(error_msg=f"Sarvam API error: {e}", exception=e) except Exception as e: - logger.error(f"Failed to connect to Sarvam: {e}") self._socket_client = None self._websocket_context = None - await self.push_error(ErrorFrame(f"Failed to connect to Sarvam: {e}")) + await self.push_error(error_msg=f"Failed to connect to Sarvam: {e}", exception=e) async def _disconnect(self): """Disconnect from Sarvam WebSocket API using SDK.""" @@ -351,7 +348,9 @@ class SarvamSTTService(STTService): # Exit the async context manager await self._websocket_context.__aexit__(None, None, None) except Exception as e: - logger.error(f"Error closing WebSocket connection: {e}") + await self.push_error( + error_msg=f"Error closing WebSocket connection: {e}", exception=e + ) finally: logger.debug("Disconnected from Sarvam WebSocket") self._socket_client = None @@ -371,8 +370,7 @@ class SarvamSTTService(STTService): # Messages will be handled via the _message_handler callback await self._socket_client.start_listening() except Exception as e: - logger.error(f"Error in Sarvam receive task: {e}") - await self.push_error(ErrorFrame(f"Sarvam receive task error: {e}")) + await self.push_error(error_msg=f"Sarvam receive task error: {e}", exception=e) async def _handle_message(self, message): """Handle incoming WebSocket message from Sarvam SDK. @@ -427,8 +425,7 @@ class SarvamSTTService(STTService): await self.stop_processing_metrics() except Exception as e: - logger.error(f"Error handling Sarvam message: {e}") - await self.push_error(ErrorFrame(f"Failed to handle message: {e}")) + await self.push_error(error_msg=f"Failed to handle message: {e}", exception=e) await self.stop_all_metrics() @traced_stt diff --git a/src/pipecat/services/sarvam/tts.py b/src/pipecat/services/sarvam/tts.py index 127a0d589..b60a505eb 100644 --- a/src/pipecat/services/sarvam/tts.py +++ b/src/pipecat/services/sarvam/tts.py @@ -254,8 +254,7 @@ class SarvamHttpTTSService(TTSService): async with self._session.post(url, json=payload, headers=headers) as response: if response.status != 200: error_text = await response.text() - logger.error(f"Sarvam API error: {error_text}") - await self.push_error(ErrorFrame(error=f"Sarvam API error: {error_text}")) + yield ErrorFrame(error=f"Sarvam API error: {error_text}") return response_data = await response.json() @@ -264,8 +263,7 @@ class SarvamHttpTTSService(TTSService): # Decode base64 audio data if "audios" not in response_data or not response_data["audios"]: - logger.error("No audio data received from Sarvam API") - await self.push_error(ErrorFrame(error="No audio data received")) + yield ErrorFrame(error="No audio data received") return # Get the first audio (there should be only one for single text input) @@ -286,8 +284,7 @@ class SarvamHttpTTSService(TTSService): yield frame except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + yield ErrorFrame(error=f"Error generating TTS: {e}", exception=e) finally: await self.stop_ttfb_metrics() yield TTSStoppedFrame() @@ -517,9 +514,11 @@ class SarvamTTSService(InterruptibleTTSService): async def process_frame(self, frame: Frame, direction: FrameDirection): """Process a frame and flush audio if it's the end of a full response.""" - if isinstance(frame, LLMFullResponseEndFrame): + await super().process_frame(frame, direction) + + # When the LLM finishes responding, flush any remaining text in Sarvam's buffer + if isinstance(frame, (LLMFullResponseEndFrame, EndFrame)): await self.flush_audio() - return await super().process_frame(frame, direction) async def _update_settings(self, settings: Mapping[str, Any]): """Update service settings and reconnect if voice changed.""" @@ -560,8 +559,7 @@ class SarvamTTSService(InterruptibleTTSService): await self._disconnect_websocket() except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) finally: # Reset state only after everything is cleaned up self._started = False @@ -585,8 +583,9 @@ class SarvamTTSService(InterruptibleTTSService): await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error( + error_msg=f"Error connecting to Sarvam TTS Websocket: {e}", exception=e + ) self._websocket = None await self._call_event_handler("on_connection_error", f"{e}") @@ -602,8 +601,7 @@ class SarvamTTSService(InterruptibleTTSService): await self._websocket.send(json.dumps(config_message)) logger.debug("Configuration sent successfully") except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) raise async def _disconnect_websocket(self): @@ -615,8 +613,7 @@ class SarvamTTSService(InterruptibleTTSService): logger.debug("Disconnecting from Sarvam") await self._websocket.close() except Exception as e: - logger.error(f"{self} error closing websocket: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Error closing websocket: {e}", exception=e) finally: self._started = False self._websocket = None @@ -640,7 +637,7 @@ class SarvamTTSService(InterruptibleTTSService): await self.push_frame(frame) elif msg.get("type") == "error": error_msg = msg["data"]["message"] - logger.error(f"TTS Error: {error_msg}") + await self.push_error(error_msg=f"TTS Error: {error_msg}") # If it's a timeout error, the connection might need to be reset if "too long" in error_msg.lower() or "timeout" in error_msg.lower(): @@ -702,13 +699,11 @@ class SarvamTTSService(InterruptibleTTSService): await self._send_text(text) await self.start_tts_usage_metrics(text) except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") yield TTSStoppedFrame() await self._disconnect() await self._connect() return yield None except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") diff --git a/src/pipecat/services/simli/video.py b/src/pipecat/services/simli/video.py index bac54f35b..b0697f34c 100644 --- a/src/pipecat/services/simli/video.py +++ b/src/pipecat/services/simli/video.py @@ -48,12 +48,14 @@ class SimliVideoService(FrameProcessor): """Input parameters for Simli video configuration. Parameters: + enable_logging: Whether to enable Simli logging. max_session_length: Absolute maximum session duration in seconds. Avatar will disconnect after this time even if it's speaking. max_idle_time: Maximum duration in seconds the avatar is not speaking before the avatar disconnects. """ + enable_logging: Optional[bool] = None max_session_length: Optional[int] = None max_idle_time: Optional[int] = None @@ -84,6 +86,10 @@ class SimliVideoService(FrameProcessor): Please use 'api_key' and 'face_id' parameters instead. use_turn_server: Whether to use TURN server for connection. Defaults to False. + + .. deprecated:: 0.0.95 + The 'use_turn_server' parameter is deprecated and will be removed in a future version. + latency_interval: Latency interval setting for sending health checks to check the latency to Simli Servers. Defaults to 0. simli_url: URL of the simli servers. Can be changed for custom deployments @@ -135,15 +141,22 @@ class SimliVideoService(FrameProcessor): config = SimliConfig(**config_kwargs) + if use_turn_server: + warnings.warn( + "The 'use_turn_server' parameter is deprecated and will be removed in a future version.", + DeprecationWarning, + stacklevel=2, + ) + self._initialized = False # Add buffer time to session limits config.maxIdleTime += 5 config.maxSessionLength += 5 self._simli_client = SimliClient( - config, - use_turn_server, - latency_interval, + config=config, + latencyInterval=latency_interval, simliURL=simli_url, + enable_logging=params.enable_logging or False, ) self._pipecat_resampler: AudioResampler = None @@ -168,7 +181,7 @@ class SimliVideoService(FrameProcessor): self._audio_task = self.create_task(self._consume_and_process_audio()) self._video_task = self.create_task(self._consume_and_process_video()) except Exception as e: - logger.error(f"{self}: unable to start connection: {e}") + await self.push_error(error_msg=f"Unable to start connection: {e}", exception=e) async def _consume_and_process_audio(self): """Consume audio frames from Simli and push them downstream.""" @@ -246,7 +259,7 @@ class SimliVideoService(FrameProcessor): await self._simli_client.send(audioBytes) return except Exception as e: - logger.exception(f"{self} exception: {e}") + await self.push_error(error_msg=f"Error sending audio: {e}", exception=e) elif isinstance(frame, TTSStoppedFrame): try: if self._previously_interrupted and len(self._audio_buffer) > 0: @@ -254,7 +267,7 @@ class SimliVideoService(FrameProcessor): self._previously_interrupted = False self._audio_buffer = bytearray() except Exception as e: - logger.exception(f"{self} exception: {e}") + await self.push_error(error_msg=f"Error stopping TTS: {e}", exception=e) return elif isinstance(frame, (EndFrame, CancelFrame)): await self._stop() diff --git a/src/pipecat/services/soniox/stt.py b/src/pipecat/services/soniox/stt.py index b4bcb7ba1..4afcaa7bb 100644 --- a/src/pipecat/services/soniox/stt.py +++ b/src/pipecat/services/soniox/stt.py @@ -194,7 +194,7 @@ class SonioxSTTService(STTService): self._websocket = await websocket_connect(self._url) if not self._websocket: - logger.error(f"Unable to connect to Soniox API at {self._url}") + await self.push_error(error_msg=f"Unable to connect to Soniox API at {self._url}") # If vad_force_turn_endpoint is not enabled, we need to enable endpoint detection. # Either one or the other is required. @@ -327,8 +327,7 @@ class SonioxSTTService(STTService): # Expected when closing the connection logger.debug("WebSocket connection closed, keepalive task stopped.") except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) async def _receive_task_handler(self): if not self._websocket: @@ -404,13 +403,8 @@ class SonioxSTTService(STTService): if error_code or error_message: # In case of error, still send the final transcript (if any remaining in the buffer). await send_endpoint_transcript() - logger.error( - f"{self} error: {error_code} (_receive_task_handler) - {error_message}" - ) await self.push_error( - ErrorFrame( - error=f"{self} error: {error_code} (_receive_task_handler) - {error_message}" - ) + error_msg=f"Error: {error_code} (_receive_task_handler) - {error_message}" ) finished = content.get("finished") @@ -425,5 +419,4 @@ class SonioxSTTService(STTService): # Expected when closing the connection. pass except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Error receiving message: {e}", exception=e) diff --git a/src/pipecat/services/speechmatics/stt.py b/src/pipecat/services/speechmatics/stt.py index 2d95bd69e..f18314d8d 100644 --- a/src/pipecat/services/speechmatics/stt.py +++ b/src/pipecat/services/speechmatics/stt.py @@ -467,8 +467,7 @@ class SpeechmaticsSTTService(STTService): await self._client.send_audio(audio) yield None except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") await self._disconnect() def update_params( @@ -514,8 +513,7 @@ class SpeechmaticsSTTService(STTService): self._client.send_message(payload), self.get_event_loop() ) except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) raise RuntimeError(f"error sending message to STT: {e}") async def _connect(self) -> None: @@ -581,8 +579,7 @@ class SpeechmaticsSTTService(STTService): logger.debug(f"{self} Connected to Speechmatics STT service") await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Error connecting to Speechmatics: {e}", exception=e) self._client = None async def _disconnect(self) -> None: @@ -596,8 +593,9 @@ class SpeechmaticsSTTService(STTService): except asyncio.TimeoutError: logger.warning(f"{self} Timeout while closing Speechmatics client connection") except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error( + error_msg=f"Error disconnecting from Speechmatics: {e}", exception=e + ) finally: self._client = None await self._call_event_handler("on_disconnected") diff --git a/src/pipecat/services/speechmatics/tts.py b/src/pipecat/services/speechmatics/tts.py index b8fe172e7..417540663 100644 --- a/src/pipecat/services/speechmatics/tts.py +++ b/src/pipecat/services/speechmatics/tts.py @@ -163,7 +163,7 @@ class SpeechmaticsTTSService(TTSService): # Report error frame yield ErrorFrame( - error=f"{self} Service unavailable [503] (attempt {attempt}, retry in {backoff_time:.2f}s)" + error=f"Service unavailable [503] (attempt {attempt}, retry in {backoff_time:.2f}s)" ) # Wait before retrying @@ -174,16 +174,13 @@ class SpeechmaticsTTSService(TTSService): except (ValueError, ArithmeticError): yield ErrorFrame( - error=f"{self} Service unavailable [503] (attempts {attempt})", - fatal=True, + error=f"Service unavailable [503] (attempts {attempt})", ) return # != 200 : Error if response.status != 200: - yield ErrorFrame( - error=f"{self} Service unavailable [{response.status}]", fatal=True - ) + yield ErrorFrame(error=f"Service unavailable [{response.status}]") return # Update Pipecat metrics @@ -225,7 +222,7 @@ class SpeechmaticsTTSService(TTSService): break except Exception as e: - yield ErrorFrame(error=f"{self}: Error generating TTS: {e}", fatal=True) + yield ErrorFrame(error=f"Error generating TTS: {e}") finally: # Emit the TTS stopped frame yield TTSStoppedFrame() diff --git a/src/pipecat/services/stt_service.py b/src/pipecat/services/stt_service.py index 6fb96c571..f81848415 100644 --- a/src/pipecat/services/stt_service.py +++ b/src/pipecat/services/stt_service.py @@ -38,7 +38,7 @@ class STTService(AIService): Event handlers: on_connected: Called when connected to the STT service. - on_connected: Called when disconnected from the STT service. + on_disconnected: Called when disconnected from the STT service. on_connection_error: Called when a connection to the STT service error occurs. Example:: @@ -329,4 +329,4 @@ class WebsocketSTTService(STTService, WebsocketService): async def _report_error(self, error: ErrorFrame): await self._call_event_handler("on_connection_error", error.error) - await self.push_error(error) + await self.push_error_frame(error) diff --git a/src/pipecat/services/tts_service.py b/src/pipecat/services/tts_service.py index f0d602a40..ca15cb2c0 100644 --- a/src/pipecat/services/tts_service.py +++ b/src/pipecat/services/tts_service.py @@ -12,6 +12,8 @@ from typing import ( Any, AsyncGenerator, AsyncIterator, + Awaitable, + Callable, Dict, List, Mapping, @@ -23,6 +25,8 @@ from typing import ( from loguru import logger from pipecat.frames.frames import ( + AggregatedTextFrame, + AggregationType, BotStartedSpeakingFrame, BotStoppedSpeakingFrame, CancelFrame, @@ -101,6 +105,16 @@ class TTSService(AIService): sample_rate: Optional[int] = None, # Text aggregator to aggregate incoming tokens and decide when to push to the TTS. text_aggregator: Optional[BaseTextAggregator] = None, + # Types of text aggregations that should not be spoken. + skip_aggregator_types: Optional[List[str]] = [], + # A list of callables to transform text before just before sending it to TTS. + # Each callable takes the aggregated text and its type, and returns the transformed text. + # To register, provide a list of tuples of (aggregation_type | '*', transform_function). + text_transforms: Optional[ + List[ + Tuple[AggregationType | str, Callable[[str, str | AggregationType], Awaitable[str]]] + ] + ] = None, # Text filter executed after text has been aggregated. text_filters: Optional[Sequence[BaseTextFilter]] = None, text_filter: Optional[BaseTextFilter] = None, @@ -120,6 +134,16 @@ class TTSService(AIService): pause_frame_processing: Whether to pause frame processing during audio generation. sample_rate: Output sample rate for generated audio. text_aggregator: Custom text aggregator for processing incoming text. + + .. deprecated:: 0.0.95 + Use an LLMTextProcessor before the TTSService for custom text aggregation. + + skip_aggregator_types: List of aggregation types that should not be spoken. + text_transforms: A list of callables to transform text before just before sending it + to TTS. Each callable takes the aggregated text and its type, and returns the + transformed text. To register, provide a list of tuples of + (aggregation_type | '*', transform_function). + text_filters: Sequence of text filters to apply after aggregation. text_filter: Single text filter (deprecated, use text_filters). @@ -142,7 +166,21 @@ class TTSService(AIService): self._voice_id: str = "" self._settings: Dict[str, Any] = {} self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator() - self._aggregated_text_includes_inter_frame_spaces: bool = False + if text_aggregator: + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Parameter 'text_aggregator' is deprecated. Use an LLMTextProcessor before the TTSService for custom text aggregation.", + DeprecationWarning, + ) + + self._skip_aggregator_types: List[str] = skip_aggregator_types or [] + self._text_transforms: List[ + Tuple[AggregationType | str, Callable[[str, AggregationType | str], Awaitable[str]]] + ] = text_transforms or [] + # TODO: Deprecate _text_filters when added to LLMTextProcessor self._text_filters: Sequence[BaseTextFilter] = text_filters or [] self._transport_destination: Optional[str] = transport_destination self._tracing_enabled: bool = False @@ -282,6 +320,39 @@ class TTSService(AIService): await self.cancel_task(self._stop_frame_task) self._stop_frame_task = None + def add_text_transformer( + self, + transform_function: Callable[[str, AggregationType | str], Awaitable[str]], + aggregation_type: AggregationType | str = "*", + ): + """Transform text for a specific aggregation type. + + Args: + transform_function: The function to apply for transformation. This function should take + the text and aggregation type as input and return the transformed text. + Ex.: async def my_transform(text: str, aggregation_type: str) -> str: + aggregation_type: The type of aggregation to transform. This value defaults to "*" indicating + the function should handle all text before sending to TTS. + """ + self._text_transforms.append((aggregation_type, transform_function)) + + def remove_text_transformer( + self, + transform_function: Callable[[str, AggregationType | str], Awaitable[str]], + aggregation_type: AggregationType | str = "*", + ): + """Remove a text transformer for a specific aggregation type. + + Args: + transform_function: The function to remove. + aggregation_type: The type of aggregation to remove the transformer for. + """ + self._text_transforms = [ + (agg_type, func) + for agg_type, func in self._text_transforms + if not (agg_type == aggregation_type and func == transform_function) + ] + async def _update_settings(self, settings: Mapping[str, Any]): for key, value in settings.items(): if key in self._settings: @@ -337,6 +408,8 @@ class TTSService(AIService): and frame.skip_tts ): await self.push_frame(frame, direction) + elif isinstance(frame, AggregatedTextFrame): + await self._push_tts_frames(frame) elif ( isinstance(frame, TextFrame) and not isinstance(frame, InterimTranscriptionFrame) @@ -352,17 +425,13 @@ class TTSService(AIService): # pause to avoid audio overlapping. await self._maybe_pause_frame_processing() - sentence = self._text_aggregator.text - includes_inter_frame_spaces = self._aggregated_text_includes_inter_frame_spaces + # Flush any remaining text (including text waiting for lookahead) + remaining = await self._text_aggregator.flush() + if remaining: + await self._push_tts_frames(AggregatedTextFrame(remaining.text, remaining.type)) # Reset aggregator state - await self._text_aggregator.reset() self._processing_text = False - self._aggregated_text_includes_inter_frame_spaces = False - - await self._push_tts_frames( - sentence, includes_inter_frame_spaces=includes_inter_frame_spaces - ) if isinstance(frame, LLMFullResponseEndFrame): if self._push_text_frames: await self.push_frame(frame, direction) @@ -372,7 +441,7 @@ class TTSService(AIService): # Store if we were processing text or not so we can set it back. processing_text = self._processing_text # Assumption: text in TTSSpeakFrame does not include inter-frame spaces - await self._push_tts_frames(frame.text, includes_inter_frame_spaces=False) + await self._push_tts_frames(AggregatedTextFrame(frame.text, AggregationType.SENTENCE)) # We pause processing incoming frames because we are sending data to # the TTS. We pause to avoid audio overlapping. await self._maybe_pause_frame_processing() @@ -462,21 +531,38 @@ class TTSService(AIService): async def _process_text_frame(self, frame: TextFrame): text: Optional[str] = None + includes_inter_frame_spaces: bool = False if not self._aggregate_sentences: text = frame.text + includes_inter_frame_spaces = frame.includes_inter_frame_spaces + aggregated_by = "token" + + if text: + logger.trace(f"Pushing TTS frames for text: {text}, {aggregated_by}") + await self._push_tts_frames( + AggregatedTextFrame(text, aggregated_by), includes_inter_frame_spaces + ) else: - text = await self._text_aggregator.aggregate(frame.text) - # Assumption: whether inter-frame spaces are included shouldn't - # change during aggregation, so we can just use the latest frame's - # value - self._aggregated_text_includes_inter_frame_spaces = frame.includes_inter_frame_spaces + async for aggregate in self._text_aggregator.aggregate(frame.text): + text = aggregate.text + aggregated_by = aggregate.type + logger.trace(f"Pushing TTS frames for text: {text}, {aggregated_by}") + await self._push_tts_frames( + AggregatedTextFrame(text, aggregated_by), includes_inter_frame_spaces + ) - if text: - await self._push_tts_frames( - text, includes_inter_frame_spaces=frame.includes_inter_frame_spaces - ) + async def _push_tts_frames( + self, src_frame: AggregatedTextFrame, includes_inter_frame_spaces: Optional[bool] = False + ): + type = src_frame.aggregated_by + text = src_frame.text + + # Skip sending to TTS if the aggregation type is in the skip list. Simply + # push the original frame downstream. + if type in self._skip_aggregator_types: + await self.push_frame(src_frame) + return - async def _push_tts_frames(self, text: str, includes_inter_frame_spaces: bool): # Remove leading newlines only text = text.lstrip("\n") @@ -492,20 +578,44 @@ class TTSService(AIService): await self.start_processing_metrics() - # Process all filter. + # Process all filters. for filter in self._text_filters: await filter.reset_interruption() text = await filter.filter(text) - if text: - await self.process_generator(self.run_tts(text)) + if not text.strip(): + await self.stop_processing_metrics() + return + + # To support use cases that may want to know the text before it's spoken, we + # push the AggregatedTextFrame version before transforming and sending to TTS. + # However, we do not want to add this text to the assistant context until it + # is spoken, so we set append_to_context to False. + src_frame.append_to_context = False + await self.push_frame(src_frame) + + # Note: Text transformations are meant to only affect the text sent to the TTS for + # TTS-specific purposes. This allows for explicit TTS modifications (e.g., inserting + # TTS supported tags for spelling or emotion or replacing an @ with "at"). For TTS + # services that support word-level timestamps, this CAN affect the resulting context + # since the TTSTextFrames are generated from the TTS output stream + transformed_text = text + for aggregation_type, transform in self._text_transforms: + if aggregation_type == type or aggregation_type == "*": + transformed_text = await transform(transformed_text, type) + await self.process_generator(self.run_tts(transformed_text)) await self.stop_processing_metrics() if self._push_text_frames: - # We send the original text after the audio. This way, if we are - # interrupted, the text is not added to the assistant context. - frame = TTSTextFrame(text) + # In TTS services that support word timestamps, the TTSTextFrames + # are pushed as words are spoken. However, in the case where the TTS service + # does not support word timestamps (i.e. _push_text_frames is True), we send + # the original (non-transformed) text after the TTS generation has completed. + # This way, if we are interrupted, the text is not added to the assistant + # context and the context that IS added does not include TTS-specific tags + # or transformations. + frame = TTSTextFrame(text, aggregated_by=type) frame.includes_inter_frame_spaces = includes_inter_frame_spaces await self.push_frame(frame) @@ -635,7 +745,7 @@ class WordTTSService(TTSService): else: # Assumption: word-by-word text frames don't include spaces, so # we can rely on the default includes_inter_frame_spaces=False - frame = TTSTextFrame(word) + frame = TTSTextFrame(word, aggregated_by=AggregationType.WORD) frame.pts = self._initial_word_timestamp + timestamp if frame: last_pts = frame.pts @@ -671,7 +781,7 @@ class WebsocketTTSService(TTSService, WebsocketService): async def _report_error(self, error: ErrorFrame): await self._call_event_handler("on_connection_error", error.error) - await self.push_error(error) + await self.push_error_frame(error) class InterruptibleTTSService(WebsocketTTSService): @@ -733,7 +843,7 @@ class WebsocketWordTTSService(WordTTSService, WebsocketService): async def _report_error(self, error: ErrorFrame): await self._call_event_handler("on_connection_error", error.error) - await self.push_error(error) + await self.push_error_frame(error) class InterruptibleWordTTSService(WebsocketWordTTSService): diff --git a/src/pipecat/services/ultravox/stt.py b/src/pipecat/services/ultravox/stt.py index 14eaebf6a..bef74f346 100644 --- a/src/pipecat/services/ultravox/stt.py +++ b/src/pipecat/services/ultravox/stt.py @@ -246,8 +246,7 @@ class UltravoxSTTService(AIService): logger.info("Model warm-up completed successfully") except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) def _generate_silent_audio(self, sample_rate=16000, duration_sec=1.0): """Generate silent audio as a numpy array. @@ -377,7 +376,7 @@ class UltravoxSTTService(AIService): if arr.size > 0: # Check if array is not empty audio_arrays.append(arr) except Exception as e: - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") # Handle numpy array data elif isinstance(f.audio, np.ndarray): if f.audio.size > 0: # Check if array is not empty @@ -437,17 +436,11 @@ class UltravoxSTTService(AIService): yield LLMFullResponseEndFrame() except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") else: - logger.error("No model available for text generation") yield ErrorFrame("No model available for text generation") except Exception as e: - logger.error(f"{self} exception: {e}") - import traceback - - logger.error(traceback.format_exc()) yield ErrorFrame(f"Error processing audio: {str(e)}") finally: self._buffer.is_processing = False diff --git a/src/pipecat/services/websocket_service.py b/src/pipecat/services/websocket_service.py index d19e6ad4d..1179c4c03 100644 --- a/src/pipecat/services/websocket_service.py +++ b/src/pipecat/services/websocket_service.py @@ -12,7 +12,7 @@ from typing import Awaitable, Callable, Optional import websockets from loguru import logger -from websockets.exceptions import ConnectionClosedOK +from websockets.exceptions import ConnectionClosedError, ConnectionClosedOK from websockets.protocol import State from pipecat.frames.frames import ErrorFrame @@ -137,6 +137,10 @@ class WebsocketService(ABC): # Normal closure, don't retry logger.debug(f"{self} connection closed normally: {e}") break + except ConnectionClosedError as e: + # Error closure, don't retry + logger.warning(f"{self} connection closed, but with an error: {e}") + break except Exception as e: message = f"{self} error receiving messages: {e}" logger.error(message) diff --git a/src/pipecat/services/whisper/base_stt.py b/src/pipecat/services/whisper/base_stt.py index 743895253..c6e6b63da 100644 --- a/src/pipecat/services/whisper/base_stt.py +++ b/src/pipecat/services/whisper/base_stt.py @@ -226,8 +226,7 @@ class BaseWhisperSTTService(SegmentedSTTService): logger.warning("Received empty transcription from API") except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") async def _transcribe(self, audio: bytes) -> Transcription: """Transcribe audio data to text. diff --git a/src/pipecat/services/whisper/stt.py b/src/pipecat/services/whisper/stt.py index 4e326e2ca..ec69b63ea 100644 --- a/src/pipecat/services/whisper/stt.py +++ b/src/pipecat/services/whisper/stt.py @@ -285,7 +285,6 @@ class WhisperSTTService(SegmentedSTTService): The service will normalize it to float32 in the range [-1, 1]. """ if not self._model: - logger.error(f"{self} error: Whisper model not available") yield ErrorFrame("Whisper model not available") return @@ -428,5 +427,4 @@ class WhisperSTTServiceMLX(WhisperSTTService): ) except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") diff --git a/src/pipecat/services/xtts/tts.py b/src/pipecat/services/xtts/tts.py index 34ddecbe2..cf98b2bdc 100644 --- a/src/pipecat/services/xtts/tts.py +++ b/src/pipecat/services/xtts/tts.py @@ -141,13 +141,8 @@ class XTTSService(TTSService): async with self._aiohttp_session.get(self._settings["base_url"] + "/studio_speakers") as r: if r.status != 200: text = await r.text() - logger.error( - f"{self} error getting studio speakers (status: {r.status}, error: {text})" - ) await self.push_error( - ErrorFrame( - error=f"Error getting studio speakers (status: {r.status}, error: {text})" - ) + error_msg=f"Error getting studio speakers (status: {r.status}, error: {text})" ) return self._studio_speakers = await r.json() @@ -186,7 +181,6 @@ class XTTSService(TTSService): async with self._aiohttp_session.post(url, json=payload) as r: if r.status != 200: text = await r.text() - logger.error(f"{self} error getting audio (status: {r.status}, error: {text})") yield ErrorFrame(error=f"Error getting audio (status: {r.status}, error: {text})") return diff --git a/src/pipecat/sync/base_notifier.py b/src/pipecat/sync/base_notifier.py index 60321e282..1e3b16969 100644 --- a/src/pipecat/sync/base_notifier.py +++ b/src/pipecat/sync/base_notifier.py @@ -6,31 +6,14 @@ """Base notifier interface for Pipecat.""" -from abc import ABC, abstractmethod +import warnings +from pipecat.utils.sync.base_notifier import BaseNotifier -class BaseNotifier(ABC): - """Abstract base class for notification mechanisms. - - Provides a standard interface for implementing notification and waiting - patterns used for event coordination and signaling between components - in the Pipecat framework. - """ - - @abstractmethod - async def notify(self): - """Send a notification signal. - - Implementations should trigger any waiting coroutines or processes - that are blocked on this notifier. - """ - pass - - @abstractmethod - async def wait(self): - """Wait for a notification signal. - - Implementations should block until a notification is received - from the corresponding notify() call. - """ - pass +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Package pipecat.sync is deprecated, use pipecat.utils.sync instead.", + DeprecationWarning, + stacklevel=2, + ) diff --git a/src/pipecat/sync/event_notifier.py b/src/pipecat/sync/event_notifier.py index 6708c2404..35fee6cb3 100644 --- a/src/pipecat/sync/event_notifier.py +++ b/src/pipecat/sync/event_notifier.py @@ -6,40 +6,14 @@ """Event-based notifier implementation using asyncio Event primitives.""" -import asyncio +import warnings -from pipecat.sync.base_notifier import BaseNotifier +from pipecat.utils.sync.event_notifier import EventNotifier - -class EventNotifier(BaseNotifier): - """Event-based notifier using asyncio.Event for task synchronization. - - Provides a simple notification mechanism where one task can signal - an event and other tasks can wait for that event to occur. The event - is automatically cleared after each wait operation. - """ - - def __init__(self): - """Initialize the event notifier. - - Creates an internal asyncio.Event for managing notifications. - """ - self._event = asyncio.Event() - - async def notify(self): - """Signal the event to notify waiting tasks. - - Sets the internal event, causing any tasks waiting on this - notifier to be awakened. - """ - self._event.set() - - async def wait(self): - """Wait for the event to be signaled. - - Blocks until another task calls notify(). Automatically clears - the event after being awakened so subsequent calls will wait - for the next notification. - """ - await self._event.wait() - self._event.clear() +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Package pipecat.sync is deprecated, use pipecat.utils.sync instead.", + DeprecationWarning, + stacklevel=2, + ) diff --git a/src/pipecat/tests/utils.py b/src/pipecat/tests/utils.py index 6ccce4b31..94b8cb1a4 100644 --- a/src/pipecat/tests/utils.py +++ b/src/pipecat/tests/utils.py @@ -203,8 +203,16 @@ async def run_test( if not isinstance(frame, EndFrame) or not send_end_frame: received_down_frames.append(frame) - print("received DOWN frames =", received_down_frames) - print("expected DOWN frames =", expected_down_frames) + down_frames_printed = "[" + for frame in received_down_frames: + down_frames_printed += f"{frame.__class__.__name__}, " + down_frames_printed += "]" + expected_frames_printed = "[" + for frame in expected_down_frames: + expected_frames_printed += f"{frame.__name__}, " + expected_frames_printed += "]" + print("received DOWN frames =", down_frames_printed) + print("expected DOWN frames =", expected_frames_printed) assert len(received_down_frames) == len(expected_down_frames) diff --git a/src/pipecat/transports/daily/transport.py b/src/pipecat/transports/daily/transport.py index 51ef637b8..b3cc8acf7 100644 --- a/src/pipecat/transports/daily/transport.py +++ b/src/pipecat/transports/daily/transport.py @@ -2506,13 +2506,10 @@ class DailyTransport(BaseTransport): async def _on_error(self, error): """Handle error events and push error frames.""" await self._call_event_handler("on_error", error) - # Push error frame to notify the pipeline - error_frame = ErrorFrame(error) - if self._input: - await self._input.push_error(error_frame) + await self._input.push_error(error_msg=error) elif self._output: - await self._output.push_error(error_frame) + await self._output.push_error(error_msg=error) else: logger.error("Both input and output are None while trying to push error") raise Exception("No valid input or output channel to push error") @@ -2568,7 +2565,7 @@ class DailyTransport(BaseTransport): except asyncio.TimeoutError: logger.error(f"Timeout handling dialin-ready event ({url})") except Exception as e: - logger.exception(f"Error handling dialin-ready event ({url}): {e}") + logger.error(f"Error handling dialin-ready event ({url}): {e}") async def _on_dialin_connected(self, data): """Handle dial-in connected events.""" diff --git a/src/pipecat/transports/livekit/utils.py b/src/pipecat/transports/livekit/utils.py new file mode 100644 index 000000000..741b820b2 --- /dev/null +++ b/src/pipecat/transports/livekit/utils.py @@ -0,0 +1,96 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""LiveKit REST Helpers. + +Methods that wrap the LiveKit API for room management. +""" + +import aiohttp + + +class LiveKitRESTHelper: + """Helper class for interacting with LiveKit's REST API. + + Provides methods for managing LiveKit rooms. + """ + + def __init__( + self, + *, + api_key: str, + api_secret: str, + api_url: str = "https://your-livekit-host.com", + aiohttp_session: aiohttp.ClientSession, + ): + """Initialize the LiveKit REST helper. + + Args: + api_key: Your LiveKit API key. + api_secret: Your LiveKit API secret. + api_url: LiveKit server URL (e.g. "https://your-livekit-host.com"). + aiohttp_session: Async HTTP session for making requests. + """ + self.api_key = api_key + self.api_secret = api_secret + self.api_url = api_url.rstrip("/") + self.aiohttp_session = aiohttp_session + + def _create_access_token(self, room_create: bool = True) -> str: + """Create a signed access token for LiveKit API authentication. + + Args: + room_create: Whether to grant roomCreate permission. + + Returns: + Signed JWT access token. + """ + import time + + import jwt + + claims = { + "iss": self.api_key, + "sub": self.api_key, + "nbf": int(time.time()), + "exp": int(time.time()) + 60, # Token valid for 60 seconds + "video": { + "roomCreate": room_create, + }, + } + + return jwt.encode(claims, self.api_secret, algorithm="HS256") + + async def delete_room_by_name(self, room_name: str) -> bool: + """Delete a LiveKit room by name. + + This will forcibly disconnect all participants currently in the room. + + Args: + room_name: Name of the room to delete. + + Returns: + True if deletion was successful. + + Raises: + Exception: If deletion fails. + """ + token = self._create_access_token(room_create=True) + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + } + + async with self.aiohttp_session.post( + f"{self.api_url}/twirp/livekit.RoomService/DeleteRoom", + headers=headers, + json={"room": room_name}, + ) as r: + if r.status != 200: + text = await r.text() + raise Exception(f"Failed to delete room [{room_name}] (status: {r.status}): {text}") + + return True diff --git a/src/pipecat/transports/smallwebrtc/connection.py b/src/pipecat/transports/smallwebrtc/connection.py index 60dd7798c..ef353417f 100644 --- a/src/pipecat/transports/smallwebrtc/connection.py +++ b/src/pipecat/transports/smallwebrtc/connection.py @@ -316,7 +316,7 @@ class SmallWebRTCConnection(BaseObject): logger.debug("Client not connected. Queuing app-message.") self._pending_app_messages.append(json_message) except Exception as e: - logger.exception(f"Error parsing JSON message {message}, {e}") + logger.error(f"Error parsing JSON message {message}, {e}") # Despite the fact that aiortc provides this listener, they don't have a status for "disconnected" # So, in case we loose connection, this event will not be triggered diff --git a/src/pipecat/transports/tavus/transport.py b/src/pipecat/transports/tavus/transport.py index 40ffeeed7..be1ff9afe 100644 --- a/src/pipecat/transports/tavus/transport.py +++ b/src/pipecat/transports/tavus/transport.py @@ -265,7 +265,7 @@ class TavusTransportClient: try: await self._client.cleanup() except Exception as e: - logger.exception(f"Exception during cleanup: {e}") + logger.error(f"Exception during cleanup: {e}") async def _on_joined(self, data): """Handle joined event.""" diff --git a/src/pipecat/utils/asyncio/task_manager.py b/src/pipecat/utils/asyncio/task_manager.py index 78c2ff0f1..b41a571c9 100644 --- a/src/pipecat/utils/asyncio/task_manager.py +++ b/src/pipecat/utils/asyncio/task_manager.py @@ -12,6 +12,7 @@ comprehensive monitoring and cleanup capabilities. """ import asyncio +import traceback from abc import ABC, abstractmethod from dataclasses import dataclass from typing import Coroutine, Dict, Optional, Sequence @@ -162,7 +163,9 @@ class TaskManager(BaseTaskManager): # Re-raise the exception to ensure the task is cancelled. raise except Exception as e: - logger.exception(f"{name}: unexpected exception: {e}") + tb = traceback.extract_tb(e.__traceback__) + last = tb[-1] + logger.error(f"{name} unexpected exception ({last.filename}:{last.lineno}): {e}") if not self._params: raise Exception("TaskManager is not setup: unable to get event loop") @@ -197,9 +200,17 @@ class TaskManager(BaseTaskManager): # Here are sure the task is cancelled properly. pass except Exception as e: - logger.exception(f"{name}: unexpected exception while cancelling task: {e}") + tb = traceback.extract_tb(e.__traceback__) + last = tb[-1] + logger.error( + f"{name} unexpected exception while cancelling task ({last.filename}:{last.lineno}): {e}" + ) except BaseException as e: - logger.critical(f"{name}: fatal base exception while cancelling task: {e}") + tb = traceback.extract_tb(e.__traceback__) + last = tb[-1] + logger.critical( + f"{name} fatal base exception while cancelling task ({last.filename}:{last.lineno}): {e}" + ) raise def current_tasks(self) -> Sequence[asyncio.Task]: diff --git a/src/pipecat/utils/base_object.py b/src/pipecat/utils/base_object.py index 4a62e5e4d..bace5e64b 100644 --- a/src/pipecat/utils/base_object.py +++ b/src/pipecat/utils/base_object.py @@ -187,7 +187,7 @@ class BaseObject(ABC): else: handler(self, *args, **kwargs) except Exception as e: - logger.exception(f"Exception in event handler {event_name}: {e}") + logger.error(f"Exception in event handler {event_name}: {e}") def _event_task_finished(self, task: asyncio.Task): """Clean up completed event handler tasks. diff --git a/src/pipecat/utils/string.py b/src/pipecat/utils/string.py index 5645d2bcf..b438dd4be 100644 --- a/src/pipecat/utils/string.py +++ b/src/pipecat/utils/string.py @@ -203,7 +203,7 @@ def parse_start_end_tags( class TextPartForConcatenation: """Class representing a part of text for concatenation with concatenate_aggregated_text. - Attributes: + Parameters: text: The text content. includes_inter_part_spaces: Whether any necessary inter-frame (leading/trailing) spaces are already included in the text. diff --git a/src/pipecat/utils/sync/__init__.py b/src/pipecat/utils/sync/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/pipecat/utils/sync/base_notifier.py b/src/pipecat/utils/sync/base_notifier.py new file mode 100644 index 000000000..60321e282 --- /dev/null +++ b/src/pipecat/utils/sync/base_notifier.py @@ -0,0 +1,36 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Base notifier interface for Pipecat.""" + +from abc import ABC, abstractmethod + + +class BaseNotifier(ABC): + """Abstract base class for notification mechanisms. + + Provides a standard interface for implementing notification and waiting + patterns used for event coordination and signaling between components + in the Pipecat framework. + """ + + @abstractmethod + async def notify(self): + """Send a notification signal. + + Implementations should trigger any waiting coroutines or processes + that are blocked on this notifier. + """ + pass + + @abstractmethod + async def wait(self): + """Wait for a notification signal. + + Implementations should block until a notification is received + from the corresponding notify() call. + """ + pass diff --git a/src/pipecat/utils/sync/event_notifier.py b/src/pipecat/utils/sync/event_notifier.py new file mode 100644 index 000000000..0a553fb1e --- /dev/null +++ b/src/pipecat/utils/sync/event_notifier.py @@ -0,0 +1,45 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Event-based notifier implementation using asyncio Event primitives.""" + +import asyncio + +from pipecat.utils.sync.base_notifier import BaseNotifier + + +class EventNotifier(BaseNotifier): + """Event-based notifier using asyncio.Event for task synchronization. + + Provides a simple notification mechanism where one task can signal + an event and other tasks can wait for that event to occur. The event + is automatically cleared after each wait operation. + """ + + def __init__(self): + """Initialize the event notifier. + + Creates an internal asyncio.Event for managing notifications. + """ + self._event = asyncio.Event() + + async def notify(self): + """Signal the event to notify waiting tasks. + + Sets the internal event, causing any tasks waiting on this + notifier to be awakened. + """ + self._event.set() + + async def wait(self): + """Wait for the event to be signaled. + + Blocks until another task calls notify(). Automatically clears + the event after being awakened so subsequent calls will wait + for the next notification. + """ + await self._event.wait() + self._event.clear() diff --git a/src/pipecat/utils/text/base_text_aggregator.py b/src/pipecat/utils/text/base_text_aggregator.py index 27e50fff5..e9e7d82c3 100644 --- a/src/pipecat/utils/text/base_text_aggregator.py +++ b/src/pipecat/utils/text/base_text_aggregator.py @@ -12,7 +12,45 @@ aggregated text should be sent for speech synthesis. """ from abc import ABC, abstractmethod -from typing import Optional +from dataclasses import dataclass +from enum import Enum +from typing import AsyncIterator, Optional + + +class AggregationType(str, Enum): + """Built-in aggregation strings.""" + + SENTENCE = "sentence" + WORD = "word" + + def __str__(self): + return self.value + + +@dataclass +class Aggregation: + """Data class representing aggregated text and its type. + + An Aggregation object is created whenever a stream of text is aggregated by + a text aggregator. It contains the aggregated text and a type indicating + the nature of the aggregation. + + Parameters: + text: The aggregated text content. + type: The type of aggregation the text represents (e.g., 'sentence', 'word', 'token', + 'my_custom_aggregation'). + """ + + text: str + type: str + + def __str__(self) -> str: + """Return a string representation of the aggregation. + + Returns: + A descriptive string showing the type and text of the aggregation. + """ + return f"Aggregation by {self.type}: {self.text}" class BaseTextAggregator(ABC): @@ -30,7 +68,7 @@ class BaseTextAggregator(ABC): @property @abstractmethod - def text(self) -> str: + def text(self) -> Aggregation: """Get the currently aggregated text. Subclasses must implement this property to return the text that has @@ -42,25 +80,43 @@ class BaseTextAggregator(ABC): pass @abstractmethod - async def aggregate(self, text: str) -> Optional[str]: - """Aggregate the specified text with the currently accumulated text. + async def aggregate(self, text: str) -> AsyncIterator[Aggregation]: + """Aggregate the specified text and yield completed aggregations. - This method should be implemented to define how the new text contributes - to the aggregation process. It returns the updated aggregated text if - it's ready to be processed, or None otherwise. + This method processes the input text character-by-character internally + and yields Aggregation objects as they complete. Subclasses should implement their specific logic for: - - How to combine new text with existing accumulated text + - How to process text character-by-character - When to consider the aggregated text ready for processing - What criteria determine text completion (e.g., sentence boundaries) + - When a completion occurs, yield an Aggregation object containing the + aggregated text (stripped of leading/trailing whitespace) and its type Args: text: The text to be aggregated. + Yields: + Aggregation objects as they complete. Each Aggregation consists of + the aggregated text (stripped of leading/trailing whitespace) and + a string indicating the type of aggregation (e.g., 'sentence', 'word', + 'token', 'my_custom_aggregation'). + """ + pass + # Make this a generator to satisfy type checker + yield # pragma: no cover + + @abstractmethod + async def flush(self) -> Optional[Aggregation]: + """Flush any pending aggregation. + + This method is called at the end of a stream (e.g., when receiving + LLMFullResponseEndFrame) to return any text that was buffered. + Returns: - The updated aggregated text if ready for processing, or None if more - text is needed before the aggregated content is ready. + An Aggregation object if there is pending text, or None if there + is no pending text. """ pass diff --git a/src/pipecat/utils/text/pattern_pair_aggregator.py b/src/pipecat/utils/text/pattern_pair_aggregator.py index ac074f2de..72cda358b 100644 --- a/src/pipecat/utils/text/pattern_pair_aggregator.py +++ b/src/pipecat/utils/text/pattern_pair_aggregator.py @@ -8,19 +8,41 @@ This module provides an aggregator that identifies and processes content between pattern pairs (like XML tags or custom delimiters) in streaming text, with -support for custom handlers and configurable pattern removal. +support for custom handlers and configurable actions for when a pattern is found. """ import re -from typing import Awaitable, Callable, Optional, Tuple +from enum import Enum +from typing import AsyncIterator, Awaitable, Callable, List, Optional, Tuple from loguru import logger -from pipecat.utils.string import match_endofsentence -from pipecat.utils.text.base_text_aggregator import BaseTextAggregator +from pipecat.utils.text.base_text_aggregator import Aggregation, AggregationType +from pipecat.utils.text.simple_text_aggregator import SimpleTextAggregator -class PatternMatch: +class MatchAction(Enum): + """Actions to take when a pattern pair is matched. + + Parameters: + REMOVE: The text along with its delimiters will be removed from the streaming text. + Sentence aggregation will continue on as if this text did not exist. + KEEP: The delimiters will be removed, but the content between them will be kept. + Sentence aggregation will continue on with the internal text included. + AGGREGATE: The delimiters will be removed and the content between will be treated + as a separate aggregation. Any text before the start of the pattern will be + returned early, whether or not a complete sentence was found. Then the pattern + will be returned. Then the aggregation will continue on sentence matching after + the closing delimiter is found. The content between the delimiters is not + aggregated by sentence. It is aggregated as one single block of text. + """ + + REMOVE = "remove" + KEEP = "keep" + AGGREGATE = "aggregate" + + +class PatternMatch(Aggregation): """Represents a matched pattern pair with its content. A PatternMatch object is created when a complete pattern pair is found @@ -29,62 +51,79 @@ class PatternMatch: content between the patterns. """ - def __init__(self, pattern_id: str, full_match: str, content: str): + def __init__(self, content: str, type: str, full_match: str): """Initialize a pattern match. Args: - pattern_id: The identifier of the matched pattern pair. + type: The type of the matched pattern pair. It should be representative + of the content type (e.g., 'sentence', 'code', 'speaker', 'custom'). full_match: The complete text including start and end patterns. content: The text content between the start and end patterns. """ - self.pattern_id = pattern_id + super().__init__(text=content, type=type) self.full_match = full_match - self.content = content def __str__(self) -> str: """Return a string representation of the pattern match. Returns: - A descriptive string showing the pattern ID and content. + A descriptive string showing the pattern type and content. """ - return f"PatternMatch(id={self.pattern_id}, content={self.content})" + return f"PatternMatch(type={self.type}, text={self.text}, full_match={self.full_match})" -class PatternPairAggregator(BaseTextAggregator): +class PatternPairAggregator(SimpleTextAggregator): """Aggregator that identifies and processes content between pattern pairs. This aggregator buffers text until it can identify complete pattern pairs (defined by start and end patterns), processes the content between these - patterns using registered handlers, and returns text at sentence boundaries. - It's particularly useful for processing structured content in streaming text, - such as XML tags, markdown formatting, or custom delimiters. + patterns using registered handlers. By default, its aggregation method + returns text at sentence boundaries, and remove the content found between + any matched patterns. However, matched patterns can also be configured to + returned as a separate aggregation object containing the content between + their start and end patterns or left in, so that only the delimiters are + removed and a callback can be triggered. + + This aggregator is particularly useful for processing structured content in + streaming text, such as XML tags, markdown formatting, or custom delimiters. The aggregator ensures that patterns spanning multiple text chunks are - correctly identified and handles cases where patterns contain sentence - boundaries. + correctly identified. """ - def __init__(self): + def __init__(self, **kwargs): """Initialize the pattern pair aggregator. Creates an empty aggregator with no patterns or handlers registered. Text buffering and pattern detection will begin when text is aggregated. """ - self._text = "" + super().__init__() self._patterns = {} self._handlers = {} + self._last_processed_position = 0 # Track where we last checked for complete patterns @property - def text(self) -> str: - """Get the currently buffered text. + def text(self) -> Aggregation: + """Get the currently aggregated text. Returns: - The current text buffer content that hasn't been processed yet. + The text that has been accumulated in the buffer. """ - return self._text + pattern_start = self._match_start_of_pattern(self._text) + stripped_text = self._text.strip() + type = ( + pattern_start[1].get("type", AggregationType.SENTENCE) + if pattern_start + else AggregationType.SENTENCE + ) + return Aggregation(text=stripped_text, type=type) - def add_pattern_pair( - self, pattern_id: str, start_pattern: str, end_pattern: str, remove_match: bool = True + def add_pattern( + self, + type: str, + start_pattern: str, + end_pattern: str, + action: MatchAction = MatchAction.REMOVE, ) -> "PatternPairAggregator": """Add a pattern pair to detect in the text. @@ -93,63 +132,118 @@ class PatternPairAggregator(BaseTextAggregator): the end pattern, and treat the content between them as a match. Args: - pattern_id: Unique identifier for this pattern pair. + type: Identifier for this pattern pair. Should be unique and ideally descriptive. + (e.g., 'code', 'speaker', 'custom'). type can not be 'sentence' or 'word' as + those are reserved for the default behavior. start_pattern: Pattern that marks the beginning of content. end_pattern: Pattern that marks the end of content. - remove_match: Whether to remove the matched content from the text. + action: What to do when a complete pattern is matched. + + - MatchAction.REMOVE: Remove the matched pattern from the text. + - MatchAction.KEEP: Keep the matched pattern in the text and treat it as normal text. This allows you to register handlers for the pattern without affecting the aggregation logic. + - MatchAction.AGGREGATE: Return the matched pattern as a separate aggregation object. Returns: Self for method chaining. """ - self._patterns[pattern_id] = { + if type in [AggregationType.SENTENCE, AggregationType.WORD]: + raise ValueError( + f"The aggregation type '{type}' is reserved for default behavior and can not be used for custom patterns." + ) + self._patterns[type] = { "start": start_pattern, "end": end_pattern, - "remove_match": remove_match, + "type": type, + "action": action, } return self + def add_pattern_pair( + self, pattern_id: str, start_pattern: str, end_pattern: str, remove_match: bool = True + ): + """Add a pattern pair to detect in the text. + + .. deprecated:: 0.0.95 + This function is deprecated and will be removed in a future version. + Use `add_pattern` with a type and MatchAction instead. + + This method calls `add_pattern` setting type with the provided pattern_id and action + to either MatchAction.REMOVE or MatchAction.KEEP based on `remove_match`. + + Args: + pattern_id: Identifier for this pattern pair. Should be unique and ideally descriptive. + (e.g., 'code', 'speaker', 'custom'). pattern_id can not be 'sentence' or 'word' + as those arereserved for the default behavior. + start_pattern: Pattern that marks the beginning of content. + end_pattern: Pattern that marks the end of content. + remove_match: If True, the matched pattern will be removed from the text. (Same as MatchAction.REMOVE) + If False, it will be kept and treated as normal text. (Same as MatchAction.KEEP) + """ + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("once") + warnings.warn( + "add_pattern_pair with a pattern_id or remove_match is deprecated and will be" + " removed in a future version. Use add_pattern with a type and MatchAction instead", + DeprecationWarning, + stacklevel=2, + ) + + action = MatchAction.REMOVE if remove_match else MatchAction.KEEP + return self.add_pattern( + type=pattern_id, + start_pattern=start_pattern, + end_pattern=end_pattern, + action=action, + ) + def on_pattern_match( - self, pattern_id: str, handler: Callable[[PatternMatch], Awaitable[None]] + self, type: str, handler: Callable[[PatternMatch], Awaitable[None]] ) -> "PatternPairAggregator": """Register a handler for when a pattern pair is matched. The handler will be called whenever a complete match for the - specified pattern ID is found in the text. + specified type is found in the text. Args: - pattern_id: ID of the pattern pair to match. + type: The type of the pattern pair to trigger the handler. handler: Async function to call when pattern is matched. The function should accept a PatternMatch object. Returns: Self for method chaining. """ - self._handlers[pattern_id] = handler + self._handlers[type] = handler return self - async def _process_complete_patterns(self, text: str) -> Tuple[str, bool]: - """Process all complete pattern pairs in the text. + async def _process_complete_patterns( + self, text: str, last_processed_position: int = 0 + ) -> Tuple[List[PatternMatch], str]: + """Process newly complete pattern pairs in the text. - Searches for all complete pattern pairs in the text, calls the - appropriate handlers, and optionally removes the matches. + Searches for pattern pairs that have been completed since last_processed_position, + calls the appropriate handlers, and optionally removes the matches. Args: text: The text to process for pattern matches. + last_processed_position: The position in text that was already processed. + Only patterns that end at or after this position will be processed. Returns: - Tuple of (processed_text, was_modified) where: + Tuple of (all_matches, processed_text) where: - - processed_text is the text after processing patterns - - was_modified indicates whether any changes were made + - all_matches is a list of all pattern matches found. Note: There really should only ever be 1. + - processed_text is the text after processing patterns. If no patterns are found, it will be the same as input text. """ + all_matches = [] processed_text = text - modified = False - for pattern_id, pattern_info in self._patterns.items(): + for type, pattern_info in self._patterns.items(): # Escape special regex characters in the patterns start = re.escape(pattern_info["start"]) end = re.escape(pattern_info["end"]) - remove_match = pattern_info["remove_match"] + action = pattern_info["action"] # Create regex to match from start pattern to end pattern # The .*? is non-greedy to handle nested patterns @@ -165,24 +259,31 @@ class PatternPairAggregator(BaseTextAggregator): # Create pattern match object pattern_match = PatternMatch( - pattern_id=pattern_id, full_match=full_match, content=content + content=content.strip(), type=type, full_match=full_match ) - # Call the appropriate handler if registered - if pattern_id in self._handlers: + # Check if this pattern was already processed + already_processed = match.end() <= last_processed_position + + # Only call handler for newly completed patterns + if not already_processed and type in self._handlers: try: - await self._handlers[pattern_id](pattern_match) + await self._handlers[type](pattern_match) except Exception as e: - logger.error(f"Error in pattern handler for {pattern_id}: {e}") + logger.error(f"Error in pattern handler for {type}: {e}") - # Remove the pattern from the text if configured - if remove_match: - processed_text = processed_text.replace(full_match, "", 1) - modified = True + # Handle pattern based on action + if action == MatchAction.REMOVE: + # Remove patterns are only removed once (when newly completed) + if not already_processed: + processed_text = processed_text.replace(full_match, "", 1) + else: + # KEEP/AGGREGATE patterns stay in all_matches + all_matches.append(pattern_match) - return processed_text, modified + return all_matches, processed_text - def _has_incomplete_patterns(self, text: str) -> bool: + def _match_start_of_pattern(self, text: str) -> Optional[Tuple[int, dict]]: """Check if text contains incomplete pattern pairs. Determines whether the text contains any start patterns without @@ -192,9 +293,10 @@ class PatternPairAggregator(BaseTextAggregator): text: The text to check for incomplete patterns. Returns: - True if there are incomplete patterns, False otherwise. + A tuple of (start_index, pattern_info) if an incomplete pattern is found, + or None if no patterns are found or all patterns are complete. """ - for pattern_id, pattern_info in self._patterns.items(): + for type, pattern_info in self._patterns.items(): start = pattern_info["start"] end = pattern_info["end"] @@ -203,59 +305,93 @@ class PatternPairAggregator(BaseTextAggregator): end_count = text.count(end) # If there are more starts than ends, we have incomplete patterns + # Again, this is written generically but there only ever should + # be one pattern active at a time, so the counts should be 0 or 1. + # Which is why we base the return on the first found. if start_count > end_count: - return True + start_index = text.find(start) + return [start_index, pattern_info] - return False - - async def aggregate(self, text: str) -> Optional[str]: - """Aggregate text and process pattern pairs. - - This method adds the new text to the buffer, processes any complete pattern - pairs, and returns processed text up to sentence boundaries if possible. - If there are incomplete patterns (start without matching end), it will - continue buffering text. - - Args: - text: New text to add to the buffer. - - Returns: - Processed text up to a sentence boundary, or None if more - text is needed to form a complete sentence or pattern. - """ - # Add new text to buffer - self._text += text - - # Process any complete patterns in the buffer - processed_text, modified = await self._process_complete_patterns(self._text) - - # Only update the buffer if modifications were made - if modified: - self._text = processed_text - - # Check if we have incomplete patterns - if self._has_incomplete_patterns(self._text): - # Still waiting for complete patterns - return None - - # Find sentence boundary if no incomplete patterns - eos_marker = match_endofsentence(self._text) - if eos_marker: - # Extract text up to the sentence boundary - result = self._text[:eos_marker] - self._text = self._text[eos_marker:] - return result - - # No complete sentence found yet return None + async def aggregate(self, text: str) -> AsyncIterator[PatternMatch]: + """Aggregate text and process pattern pairs. + + Processes the input text character-by-character, handles pattern pairs, + and uses the parent's lookahead logic for sentence detection when no + patterns are active. + + Args: + text: Text to aggregate. + + Yields: + PatternMatch objects as patterns complete or sentences are detected. + """ + # Process text character by character + for char in text: + self._text += char + + # Process any newly complete patterns in the buffer + # Only patterns that complete after _last_processed_position will trigger handlers + patterns, processed_text = await self._process_complete_patterns( + self._text, self._last_processed_position + ) + + # Update the last processed position to prevent re-processing patterns + # This tracks where in the buffer we've already called handlers, so we + # only trigger handlers once when a pattern completes + self._last_processed_position = len(self._text) + + self._text = processed_text + + if len(patterns) > 0: + if len(patterns) > 1: + logger.warning( + f"Multiple patterns matched: {[p.type for p in patterns]}. Only the first pattern will be returned." + ) + # If the pattern found is set to be aggregated, return it + action = self._patterns[patterns[0].type].get("action", MatchAction.REMOVE) + if action == MatchAction.AGGREGATE: + self._text = "" + yield patterns[0] + continue + + # Check if we have incomplete patterns + pattern_start = self._match_start_of_pattern(self._text) + if pattern_start is not None: + # If the start pattern is at the beginning or should not be separately aggregated, continue + if ( + pattern_start[0] == 0 + or pattern_start[1].get("action", MatchAction.REMOVE) != MatchAction.AGGREGATE + ): + continue + # For AGGREGATE patterns: yield any text before the pattern starts + # This ensures text doesn't get stuck in the buffer waiting for sentence + # boundaries when a pattern begins (e.g., "Here is code ..." yields "Here is code") + result = self._text[: pattern_start[0]] + self._text = self._text[pattern_start[0] :] + yield PatternMatch( + content=result.strip(), type=AggregationType.SENTENCE, full_match=result + ) + continue + + # Use parent's lookahead logic for sentence detection + aggregation = await super()._check_sentence_with_lookahead(char) + if aggregation: + # Convert to PatternMatch for consistency with return type + yield PatternMatch( + content=aggregation.text, type=aggregation.type, full_match=aggregation.text + ) + async def handle_interruption(self): - """Handle interruptions by clearing the buffer. + """Handle interruptions by clearing the buffer and pattern state. Called when an interruption occurs in the processing pipeline, to reset the state and discard any partially aggregated text. """ - self._text = "" + await super().handle_interruption() + self._last_processed_position = 0 + # Pattern and handler state persists across interruptions async def reset(self): """Clear the internally aggregated text. @@ -263,4 +399,6 @@ class PatternPairAggregator(BaseTextAggregator): Resets the aggregator to its initial state, discarding any buffered text and clearing pattern tracking state. """ - self._text = "" + await super().reset() + self._last_processed_position = 0 + # Pattern and handler state persists across resets diff --git a/src/pipecat/utils/text/simple_text_aggregator.py b/src/pipecat/utils/text/simple_text_aggregator.py index f9eb7d83a..1d123e8fb 100644 --- a/src/pipecat/utils/text/simple_text_aggregator.py +++ b/src/pipecat/utils/text/simple_text_aggregator.py @@ -11,10 +11,10 @@ until it finds an end-of-sentence marker, making it suitable for basic TTS text processing scenarios. """ -from typing import Optional +from typing import AsyncIterator, Optional -from pipecat.utils.string import match_endofsentence -from pipecat.utils.text.base_text_aggregator import BaseTextAggregator +from pipecat.utils.string import SENTENCE_ENDING_PUNCTUATION, match_endofsentence +from pipecat.utils.text.base_text_aggregator import Aggregation, AggregationType, BaseTextAggregator class SimpleTextAggregator(BaseTextAggregator): @@ -31,40 +31,100 @@ class SimpleTextAggregator(BaseTextAggregator): Creates an empty text buffer ready to begin accumulating text tokens. """ self._text = "" + self._needs_lookahead: bool = False @property - def text(self) -> str: + def text(self) -> Aggregation: """Get the currently aggregated text. Returns: The text that has been accumulated in the buffer. """ - return self._text + return Aggregation(text=self._text.strip(), type=AggregationType.SENTENCE) - async def aggregate(self, text: str) -> Optional[str]: - """Aggregate text and return completed sentences. + async def aggregate(self, text: str) -> AsyncIterator[Aggregation]: + """Aggregate text and yield completed sentences. - Adds the new text to the buffer and checks for end-of-sentence markers. - When a sentence boundary is found, returns the completed sentence and - removes it from the buffer. + Processes the input text character-by-character. When sentence-ending + punctuation is detected, it waits for non-whitespace lookahead before + calling NLTK. This prevents false positives like "$29." being detected + as a sentence when it's actually "$29.95". Args: - text: New text to add to the aggregation buffer. + text: Text to aggregate. + + Yields: + Complete sentences as Aggregation objects. + """ + # Process text character by character + for char in text: + self._text += char + + # Check for sentence with lookahead + result = await self._check_sentence_with_lookahead(char) + if result: + yield result + + async def _check_sentence_with_lookahead(self, char: str) -> Optional[Aggregation]: + """Check for sentence boundaries using lookahead logic. + + This method implements the core sentence detection logic with lookahead. + When sentence-ending punctuation is detected, it waits for the next + non-whitespace character before calling NLTK. This disambiguates cases + like "$29." (not a sentence) vs "$29. Next" (sentence ends at period). + Whitespace alone is not meaningful lookahead since it appears in both + cases. Instead, the first non-whitespace character after the punctuation + is used to confirm the sentence boundary. + + Subclasses can call this via super() to reuse the lookahead behavior + while adding their own logic (e.g., tag handling, pattern matching). + + Args: + char: The most recently added character (used for lookahead check). Returns: - A complete sentence if an end-of-sentence marker is found, - or None if more text is needed to complete a sentence. + Aggregation if sentence found, None otherwise. """ - result: Optional[str] = None + # If we need lookahead, check if we now have non-whitespace + if self._needs_lookahead: + # Check if the new character is non-whitespace + if char.strip(): + # We have meaningful lookahead, call NLTK + self._needs_lookahead = False + eos_marker = match_endofsentence(self._text) - self._text += text + if eos_marker: + # NLTK confirmed a sentence - return it + result = self._text[:eos_marker] + self._text = self._text[eos_marker:] + return Aggregation(text=result, type=AggregationType.SENTENCE) + # No sentence found - keep accumulating + return None + # Still whitespace, keep waiting + return None - eos_end_marker = match_endofsentence(self._text) - if eos_end_marker: - result = self._text[:eos_end_marker] - self._text = self._text[eos_end_marker:] + # Check if we just added sentence-ending punctuation + if self._text and self._text[-1] in SENTENCE_ENDING_PUNCTUATION: + # Mark that we need lookahead (don't call NLTK yet) + self._needs_lookahead = True - return result + return None + + async def flush(self) -> Optional[Aggregation]: + """Flush any remaining text in the buffer. + + Returns any text remaining in the buffer. This is called at the end + of a stream to ensure all text is processed. + + Returns: + Any remaining text as a sentence, or None if buffer is empty. + """ + if self._text: + # Return whatever we have in the buffer + result = self._text + await self.reset() + return Aggregation(text=result.strip(), type=AggregationType.SENTENCE) + return None async def handle_interruption(self): """Handle interruptions by clearing the text buffer. @@ -73,6 +133,7 @@ class SimpleTextAggregator(BaseTextAggregator): discarding any partially accumulated text. """ self._text = "" + self._needs_lookahead = False async def reset(self): """Clear the internally aggregated text. @@ -81,3 +142,4 @@ class SimpleTextAggregator(BaseTextAggregator): any accumulated text content. """ self._text = "" + self._needs_lookahead = False diff --git a/src/pipecat/utils/text/skip_tags_aggregator.py b/src/pipecat/utils/text/skip_tags_aggregator.py index 6f6f8455c..e82444ea6 100644 --- a/src/pipecat/utils/text/skip_tags_aggregator.py +++ b/src/pipecat/utils/text/skip_tags_aggregator.py @@ -11,13 +11,14 @@ between specified start/end tag pairs, ensuring that tagged content is processed as a unit regardless of internal punctuation. """ -from typing import Optional, Sequence +from typing import AsyncIterator, Optional, Sequence -from pipecat.utils.string import StartEndTags, match_endofsentence, parse_start_end_tags -from pipecat.utils.text.base_text_aggregator import BaseTextAggregator +from pipecat.utils.string import StartEndTags, parse_start_end_tags +from pipecat.utils.text.base_text_aggregator import Aggregation, AggregationType +from pipecat.utils.text.simple_text_aggregator import SimpleTextAggregator -class SkipTagsAggregator(BaseTextAggregator): +class SkipTagsAggregator(SimpleTextAggregator): """Aggregator that prevents end of sentence matching between start/end tags. This aggregator buffers text until it finds an end of sentence or a start @@ -37,66 +38,59 @@ class SkipTagsAggregator(BaseTextAggregator): tags: Sequence of StartEndTags objects defining the tag pairs that should prevent sentence boundary detection. """ - self._text = "" + super().__init__() self._tags = tags self._current_tag: Optional[StartEndTags] = None self._current_tag_index: int = 0 - @property - def text(self) -> str: - """Get the currently buffered text. - - Returns: - The current text buffer content that hasn't been processed yet. - """ - return self._text - - async def aggregate(self, text: str) -> Optional[str]: + async def aggregate(self, text: str) -> AsyncIterator[Aggregation]: """Aggregate text while respecting tag boundaries. - This method adds the new text to the buffer, processes any complete - pattern pairs, and returns processed text up to sentence boundaries if - possible. If there are incomplete patterns (start without matching - end), it will continue buffering text. + Processes the input text character-by-character, updates tag state, and + uses the parent's lookahead logic for sentence detection when not + inside tags. Args: - text: New text to add to the buffer. + text: Text to aggregate. - Returns: - Processed text up to a sentence boundary (when not within tags), - or None if more text is needed to complete a sentence or close tags. + Yields: + Aggregation objects containing text up to a sentence boundary, + marked as SENTENCE type. """ - # Add new text to buffer - self._text += text + # Process text character by character + for char in text: + self._text += char - (self._current_tag, self._current_tag_index) = parse_start_end_tags( - self._text, self._tags, self._current_tag, self._current_tag_index - ) + # Update tag state + (self._current_tag, self._current_tag_index) = parse_start_end_tags( + self._text, self._tags, self._current_tag, self._current_tag_index + ) - # Find sentence boundary if no incomplete patterns - if not self._current_tag: - eos_marker = match_endofsentence(self._text) - if eos_marker: - # Extract text up to the sentence boundary - result = self._text[:eos_marker] - self._text = self._text[eos_marker:] - return result + # If inside tags, don't check for sentences + if self._current_tag: + continue - # No complete sentence found yet - return None + # Otherwise, use parent's lookahead logic for sentence detection + result = await super()._check_sentence_with_lookahead(char) + if result: + yield result async def handle_interruption(self): - """Handle interruptions by clearing the buffer. + """Handle interruptions by clearing the buffer and tag state. Called when an interruption occurs in the processing pipeline, to reset the state and discard any partially aggregated text. """ - self._text = "" + await super().handle_interruption() + self._current_tag = None + self._current_tag_index = 0 async def reset(self): - """Clear the internally aggregated text. + """Clear the internally aggregated text and tag state. Resets the aggregator to its initial state, discarding any buffered text. """ - self._text = "" + await super().reset() + self._current_tag = None + self._current_tag_index = 0 diff --git a/src/pipecat/utils/tracing/service_decorators.py b/src/pipecat/utils/tracing/service_decorators.py index ead4aa9e8..af9985d54 100644 --- a/src/pipecat/utils/tracing/service_decorators.py +++ b/src/pipecat/utils/tracing/service_decorators.py @@ -92,6 +92,24 @@ def _add_token_usage_to_span(span, token_usage): span.set_attribute("gen_ai.usage.input_tokens", token_usage["prompt_tokens"]) if "completion_tokens" in token_usage: span.set_attribute("gen_ai.usage.output_tokens", token_usage["completion_tokens"]) + # Add cached token metrics for dictionary + if ( + "cache_read_input_tokens" in token_usage + and token_usage["cache_read_input_tokens"] is not None + ): + span.set_attribute( + "gen_ai.usage.cache_read_input_tokens", token_usage["cache_read_input_tokens"] + ) + if ( + "cache_creation_input_tokens" in token_usage + and token_usage["cache_creation_input_tokens"] is not None + ): + span.set_attribute( + "gen_ai.usage.cache_creation_input_tokens", + token_usage["cache_creation_input_tokens"], + ) + if "reasoning_tokens" in token_usage and token_usage["reasoning_tokens"] is not None: + span.set_attribute("gen_ai.usage.reasoning_tokens", token_usage["reasoning_tokens"]) else: # Handle LLMTokenUsage object span.set_attribute("gen_ai.usage.input_tokens", getattr(token_usage, "prompt_tokens", 0)) @@ -99,6 +117,19 @@ def _add_token_usage_to_span(span, token_usage): "gen_ai.usage.output_tokens", getattr(token_usage, "completion_tokens", 0) ) + # Add cached token metrics for LLMTokenUsage object + cache_read_tokens = getattr(token_usage, "cache_read_input_tokens", None) + if cache_read_tokens is not None: + span.set_attribute("gen_ai.usage.cache_read_input_tokens", cache_read_tokens) + + cache_creation_tokens = getattr(token_usage, "cache_creation_input_tokens", None) + if cache_creation_tokens is not None: + span.set_attribute("gen_ai.usage.cache_creation_input_tokens", cache_creation_tokens) + + reasoning_tokens = getattr(token_usage, "reasoning_tokens", None) + if reasoning_tokens is not None: + span.set_attribute("gen_ai.usage.reasoning_tokens", reasoning_tokens) + def traced_tts(func: Optional[Callable] = None, *, name: Optional[str] = None) -> Callable: """Trace TTS service methods with TTS-specific attributes. @@ -452,7 +483,9 @@ def traced_llm(func: Optional[Callable] = None, *, name: Optional[str] = None) - # Add all available attributes to the span attribute_kwargs = { "service_name": service_class_name, - "model": getattr(self, "model_name", "unknown"), + "model": getattr( + self, getattr(self, "_full_model_name", "model_name"), "unknown" + ), "stream": True, # Most LLM services use streaming "parameters": params, } @@ -715,7 +748,7 @@ def traced_gemini_live(operation: str) -> Callable: else: operation_attrs["tool.result_status"] = "completed" - except json.JSONDecodeError as e: + except json.JSONDecodeError: operation_attrs["tool.result"] = ( f"Invalid JSON: {str(result_content)[:500]}" ) diff --git a/tests/test_frame_processor.py b/tests/test_frame_processor.py index d0072e5fb..2a3e1e66c 100644 --- a/tests/test_frame_processor.py +++ b/tests/test_frame_processor.py @@ -6,13 +6,17 @@ import asyncio import unittest +from dataclasses import dataclass from pipecat.frames.frames import ( + DataFrame, EndFrame, Frame, InterruptionFrame, OutputTransportMessageUrgentFrame, + SystemFrame, TextFrame, + UninterruptibleFrame, ) from pipecat.pipeline.pipeline import Pipeline from pipecat.processors.filters.identity_filter import IdentityFilter @@ -110,3 +114,75 @@ class TestFrameProcessor(unittest.IsolatedAsyncioTestCase): expected_down_frames=expected_down_frames, send_end_frame=False, ) + + async def test_interruptible_frames(self): + @dataclass + class TestInterruptibleFrame(DataFrame): + text: str + + class DelayTestFrameProcessor(FrameProcessor): + """This processor just delays processing frames so we have time to + try to interrupt them. + """ + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if not isinstance(frame, SystemFrame): + # Sleep more than SleepFrame default. + await asyncio.sleep(0.4) + await self.push_frame(frame, direction) + + pipeline = Pipeline([DelayTestFrameProcessor()]) + + frames_to_send = [ + TestInterruptibleFrame(text="Hello from Pipecat!"), + # Make sure we hit the DelayTestFrameProcessor first. + SleepFrame(), + # Just a random interruption. This should cause the interruption of + # TestInterruptibleFrame. + InterruptionFrame(), + ] + expected_down_frames = [InterruptionFrame] + await run_test( + pipeline, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + + async def test_uninterruptible_frames(self): + @dataclass + class TestUninterruptibleFrame(DataFrame, UninterruptibleFrame): + text: str + + class DelayTestFrameProcessor(FrameProcessor): + """This processor just delays processing non-InterruptionFrame so we + have time to try to interrupt them. + + """ + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if not isinstance(frame, SystemFrame): + # Sleep more than SleepFrame default. + await asyncio.sleep(0.4) + await self.push_frame(frame, direction) + + pipeline = Pipeline([DelayTestFrameProcessor()]) + + frames_to_send = [ + TestUninterruptibleFrame(text="Hello from Pipecat!"), + # Make sure we hit the DelayTestFrameProcessor first. + SleepFrame(), + # Just a random interruption. This should not cause the interruption + # of TestUninterruptibleFrame. + InterruptionFrame(), + ] + expected_down_frames = [ + InterruptionFrame, + TestUninterruptibleFrame, + ] + await run_test( + pipeline, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) diff --git a/tests/test_ivr_navigation.py b/tests/test_ivr_navigation.py index 61464fb32..74bbb6266 100644 --- a/tests/test_ivr_navigation.py +++ b/tests/test_ivr_navigation.py @@ -10,6 +10,7 @@ from unittest.mock import AsyncMock from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.extensions.ivr.ivr_navigator import IVRProcessor from pipecat.frames.frames import ( + LLMFullResponseEndFrame, LLMMessagesUpdateFrame, LLMTextFrame, OutputDTMFUrgentFrame, @@ -334,10 +335,12 @@ class TestIVRNavigation(unittest.IsolatedAsyncioTestCase): frames_to_send = [ LLMTextFrame(text="Hello, I'm trying to reach billing."), + LLMFullResponseEndFrame(), ] expected_down_frames = [ LLMTextFrame, # Should pass through unchanged + LLMFullResponseEndFrame, ] expected_up_frames = [ diff --git a/tests/test_pattern_pair_aggregator.py b/tests/test_pattern_pair_aggregator.py index 8426dcf39..4726e7b81 100644 --- a/tests/test_pattern_pair_aggregator.py +++ b/tests/test_pattern_pair_aggregator.py @@ -7,141 +7,188 @@ import unittest from unittest.mock import AsyncMock -from pipecat.utils.text.pattern_pair_aggregator import PatternMatch, PatternPairAggregator +from pipecat.utils.text.pattern_pair_aggregator import ( + MatchAction, + PatternMatch, + PatternPairAggregator, +) class TestPatternPairAggregator(unittest.IsolatedAsyncioTestCase): def setUp(self): self.aggregator = PatternPairAggregator() self.test_handler = AsyncMock() + self.code_handler = AsyncMock() # Add a test pattern self.aggregator.add_pattern_pair( pattern_id="test_pattern", start_pattern="", end_pattern="", - remove_match=True, + ) + self.aggregator.add_pattern( + type="code_pattern", + start_pattern="", + end_pattern="", + action=MatchAction.AGGREGATE, ) # Register the mock handler self.aggregator.on_pattern_match("test_pattern", self.test_handler) + self.aggregator.on_pattern_match("code_pattern", self.code_handler) async def test_pattern_match_and_removal(self): - # First part doesn't complete the pattern - result = await self.aggregator.aggregate("Hello pattern") - self.assertIsNone(result) - self.assertEqual(self.aggregator.text, "Hello pattern") - - # Second part completes the pattern and includes an exclamation point - result = await self.aggregator.aggregate(" content!") + text = "Hello pattern content!" + results = [result async for result in self.aggregator.aggregate(text)] # Verify the handler was called with correct PatternMatch object self.test_handler.assert_called_once() call_args = self.test_handler.call_args[0][0] self.assertIsInstance(call_args, PatternMatch) - self.assertEqual(call_args.pattern_id, "test_pattern") + self.assertEqual(call_args.type, "test_pattern") self.assertEqual(call_args.full_match, "pattern content") - self.assertEqual(call_args.content, "pattern content") + self.assertEqual(call_args.text, "pattern content") - # The exclamation point should be treated as a sentence boundary, - # so the result should include just text up to and including "!" - self.assertEqual(result, "Hello !") + # No results yet (waiting for lookahead after "!") + self.assertEqual(len(results), 0) - # Next sentence should be processed separately - result = await self.aggregator.aggregate(" This is another sentence.") - self.assertEqual(result, " This is another sentence.") + # Next sentence should provide the lookahead and trigger the previous sentence + async for result in self.aggregator.aggregate(" This is another sentence."): + results.append(result) + + # First result should be "Hello !" triggered by the space lookahead + self.assertEqual(len(results), 1) + self.assertEqual(results[0].text, "Hello !") + self.assertEqual(results[0].type, "sentence") + + # Now flush to get the remaining sentence + result = await self.aggregator.flush() + self.assertEqual(result.text, "This is another sentence.") # Buffer should be empty after returning a complete sentence - self.assertEqual(self.aggregator.text, "") + self.assertEqual(self.aggregator.text.text, "") + + async def test_pattern_match_and_aggregate(self): + text = "Here is code pattern content This is another sentence." + + results = [result async for result in self.aggregator.aggregate(text)] + + # First result should be "Here is code" when pattern starts + self.assertEqual(results[0].text, "Here is code") + self.assertEqual(results[0].type, "sentence") + + # Second result should be the code pattern content + self.assertEqual(results[1].text, "pattern content") + self.assertEqual(results[1].type, "code_pattern") + + # Verify the handler was called with correct PatternMatch object + self.code_handler.assert_called_once() + call_args = self.code_handler.call_args[0][0] + self.assertIsInstance(call_args, PatternMatch) + self.assertEqual(call_args.type, "code_pattern") + self.assertEqual(call_args.full_match, "pattern content") + self.assertEqual(call_args.text, "pattern content") + + # Last sentence needs flush (waiting for lookahead after ".") + result = await self.aggregator.flush() + self.assertEqual(result.text, "This is another sentence.") + self.assertEqual(result.type, "sentence") + + # Buffer should be empty after returning a complete sentence + self.assertEqual(self.aggregator.text.text, "") async def test_incomplete_pattern(self): - # Add text with incomplete pattern - result = await self.aggregator.aggregate("Hello pattern content") - + text = "Hello pattern content" + results = [result async for result in self.aggregator.aggregate(text)] # No complete pattern yet, so nothing should be returned - self.assertIsNone(result) + self.assertEqual(len(results), 0) # The handler should not be called yet self.test_handler.assert_not_called() # Buffer should contain the incomplete text - self.assertEqual(self.aggregator.text, "Hello pattern content") + self.assertEqual(self.aggregator.text.text, "Hello pattern content") + self.assertEqual(self.aggregator.text.type, "test_pattern") # Reset and confirm buffer is cleared await self.aggregator.reset() - self.assertEqual(self.aggregator.text, "") + self.assertEqual(self.aggregator.text.text, "") async def test_multiple_patterns(self): # Set up multiple patterns and handlers voice_handler = AsyncMock() emphasis_handler = AsyncMock() - self.aggregator.add_pattern_pair( - pattern_id="voice", start_pattern="", end_pattern="", remove_match=True + self.aggregator.add_pattern( + type="voice", + start_pattern="", + end_pattern="", + action=MatchAction.REMOVE, ) - self.aggregator.add_pattern_pair( - pattern_id="emphasis", + self.aggregator.add_pattern( + type="emphasis", start_pattern="", end_pattern="", - remove_match=False, # Keep emphasis tags + action=MatchAction.KEEP, # Keep emphasis tags ) self.aggregator.on_pattern_match("voice", voice_handler) self.aggregator.on_pattern_match("emphasis", emphasis_handler) - # Test with multiple patterns in one text block text = "Hello female I am very excited to meet you!" - result = await self.aggregator.aggregate(text) + results = [result async for result in self.aggregator.aggregate(text)] # Both handlers should be called with correct data voice_handler.assert_called_once() voice_match = voice_handler.call_args[0][0] - self.assertEqual(voice_match.pattern_id, "voice") - self.assertEqual(voice_match.content, "female") + self.assertEqual(voice_match.type, "voice") + self.assertEqual(voice_match.text, "female") emphasis_handler.assert_called_once() emphasis_match = emphasis_handler.call_args[0][0] - self.assertEqual(emphasis_match.pattern_id, "emphasis") - self.assertEqual(emphasis_match.content, "very") + self.assertEqual(emphasis_match.type, "emphasis") + self.assertEqual(emphasis_match.text, "very") + # With lookahead, we need to flush to get the final sentence + self.assertEqual(len(results), 0) # Waiting for lookahead after "!" + + result = await self.aggregator.flush() # Voice pattern should be removed, emphasis pattern should remain - self.assertEqual(result, "Hello I am very excited to meet you!") + self.assertEqual(result.text, "Hello I am very excited to meet you!") # Buffer should be empty - self.assertEqual(self.aggregator.text, "") + self.assertEqual(self.aggregator.text.text, "") async def test_handle_interruption(self): - # Start with incomplete pattern - result = await self.aggregator.aggregate("Hello pattern") - self.assertIsNone(result) + text = "Hello pattern" + results = [result async for result in self.aggregator.aggregate(text)] + self.assertEqual(len(results), 0) # Simulate interruption await self.aggregator.handle_interruption() # Buffer should be cleared - self.assertEqual(self.aggregator.text, "") + self.assertEqual(self.aggregator.text.text, "") # Handler should not have been called self.test_handler.assert_not_called() async def test_pattern_across_sentences(self): - # Test pattern that spans multiple sentences - result = await self.aggregator.aggregate("Hello This is sentence one.") - - # First sentence contains start of pattern but no end, so no complete pattern yet - self.assertIsNone(result) - - # Add second part with pattern end - result = await self.aggregator.aggregate(" This is sentence two. Final sentence.") + text = "Hello This is sentence one. This is sentence two. Final sentence." + results = [result async for result in self.aggregator.aggregate(text)] # Handler should be called with entire content self.test_handler.assert_called_once() call_args = self.test_handler.call_args[0][0] - self.assertEqual(call_args.content, "This is sentence one. This is sentence two.") + self.assertEqual(call_args.text, "This is sentence one. This is sentence two.") + # With lookahead, we need to flush to get the final sentence + self.assertEqual(len(results), 0) # Waiting for lookahead after "." + + result = await self.aggregator.flush() # Pattern should be removed, resulting in text with sentences merged - self.assertEqual(result, "Hello Final sentence.") + self.assertEqual(result.text, "Hello Final sentence.") # Buffer should be empty - self.assertEqual(self.aggregator.text, "") + self.assertEqual(self.aggregator.text.text, "") diff --git a/tests/test_piper_tts.py b/tests/test_piper_tts.py index 75893f93f..a006f555c 100644 --- a/tests/test_piper_tts.py +++ b/tests/test_piper_tts.py @@ -13,6 +13,7 @@ import pytest from aiohttp import web from pipecat.frames.frames import ( + AggregatedTextFrame, ErrorFrame, TTSAudioRawFrame, TTSSpeakFrame, @@ -74,6 +75,7 @@ async def test_run_piper_tts_success(aiohttp_client): ] expected_returned_frames = [ + AggregatedTextFrame, TTSStartedFrame, TTSAudioRawFrame, TTSAudioRawFrame, @@ -121,7 +123,7 @@ async def test_run_piper_tts_error(aiohttp_client): TTSSpeakFrame(text="Error case."), ] - expected_down_frames = [TTSStoppedFrame, TTSTextFrame] + expected_down_frames = [AggregatedTextFrame, TTSStoppedFrame, TTSTextFrame] expected_up_frames = [ErrorFrame] diff --git a/tests/test_simple_text_aggregator.py b/tests/test_simple_text_aggregator.py index ff6dd1847..7b87c551c 100644 --- a/tests/test_simple_text_aggregator.py +++ b/tests/test_simple_text_aggregator.py @@ -14,16 +14,112 @@ class TestSimpleTextAggregator(unittest.IsolatedAsyncioTestCase): self.aggregator = SimpleTextAggregator() async def test_reset_aggregations(self): - assert await self.aggregator.aggregate("Hello ") == None - assert self.aggregator.text == "Hello " + text = "Hello " + results = [agg async for agg in self.aggregator.aggregate(text)] + + # No complete sentences yet + assert len(results) == 0 + assert self.aggregator.text.text == "Hello" await self.aggregator.reset() - assert self.aggregator.text == "" + assert self.aggregator.text.text == "" async def test_simple_sentence(self): - assert await self.aggregator.aggregate("Hello ") == None - assert await self.aggregator.aggregate("Pipecat!") == "Hello Pipecat!" - assert self.aggregator.text == "" + text = "Hello Pipecat!" + results = [agg async for agg in self.aggregator.aggregate(text)] + + # No complete sentences yet (waiting for lookahead after "!") + assert len(results) == 0 + + # Flush to get the pending sentence + aggregate = await self.aggregator.flush() + assert aggregate.text == "Hello Pipecat!" + assert aggregate.type == "sentence" + assert self.aggregator.text.text == "" async def test_multiple_sentences(self): - assert await self.aggregator.aggregate("Hello Pipecat! How are ") == "Hello Pipecat!" - assert await self.aggregator.aggregate("you?") == " How are you?" + text = "Hello Pipecat! How are you?" + results = [agg async for agg in self.aggregator.aggregate(text)] + + # First sentence should be complete (lookahead from "H" confirmed it) + assert len(results) == 1 + assert results[0].text == "Hello Pipecat!" + + # Flush to get the pending sentence + result = await self.aggregator.flush() + assert result.text == "How are you?" + + async def test_lookahead_decimal_number(self): + """Test that $29.95 is not split at $29.""" + text = "Ask me for only $29.95/month." + results = [agg async for agg in self.aggregator.aggregate(text)] + + # No complete sentences yet (waiting for lookahead after final ".") + assert len(results) == 0 + + # Can use flush() to get the pending sentence at end of stream + result = await self.aggregator.flush() + assert result.text == "Ask me for only $29.95/month." + + async def test_lookahead_abbreviation(self): + """Test that Mr. Smith is not split at Mr.""" + text = "Hello Mr. Smith." + results = [agg async for agg in self.aggregator.aggregate(text)] + + # No complete sentences yet (waiting for lookahead after final ".") + assert len(results) == 0 + + # Can use flush() to get the pending sentence at end of stream + result = await self.aggregator.flush() + assert result.text == "Hello Mr. Smith." + + async def test_lookahead_actual_sentence_end(self): + """Test that a real sentence end is detected after lookahead.""" + text = "Hello world. Next sentence" + results = [agg async for agg in self.aggregator.aggregate(text)] + + # First sentence should be complete (lookahead from "N" confirmed it) + assert len(results) == 1 + assert results[0].text == "Hello world." + + async def test_flush_pending_sentence(self): + """Test that flush() returns pending sentence waiting for lookahead.""" + text = "Hello world." + results = [agg async for agg in self.aggregator.aggregate(text)] + + # No complete sentences yet (waiting for lookahead) + assert len(results) == 0 + + # Call flush to get it + result = await self.aggregator.flush() + assert result is not None + assert result.text == "Hello world." + # Flush again should return None + assert await self.aggregator.flush() == None + + async def test_flush_with_no_pending(self): + """Test that flush() returns any remaining text in buffer.""" + text = "Hello" + results = [agg async for agg in self.aggregator.aggregate(text)] + + # No complete sentences + assert len(results) == 0 + + result = await self.aggregator.flush() + # flush() now returns any remaining text, not just pending lookahead + assert result is not None + assert result.text == "Hello" + # Buffer should be empty after flush + assert self.aggregator.text.text == "" + + async def test_flush_after_lookahead_confirmed(self): + """Test flush after lookahead has already confirmed sentence.""" + text = "Hello. W" + results = [agg async for agg in self.aggregator.aggregate(text)] + + # First sentence should be complete (lookahead from "W" confirmed it) + assert len(results) == 1 + assert results[0].text == "Hello." + + # flush() returns any remaining text (the "W" in this case) + result = await self.aggregator.flush() + assert result.text == "W" diff --git a/tests/test_skip_tags_aggregator.py b/tests/test_skip_tags_aggregator.py index f6cbb7b93..ffed966e1 100644 --- a/tests/test_skip_tags_aggregator.py +++ b/tests/test_skip_tags_aggregator.py @@ -17,38 +17,48 @@ class TestSkipTagsAggregator(unittest.IsolatedAsyncioTestCase): await self.aggregator.reset() # No tags involved, aggregate at end of sentence. - result = await self.aggregator.aggregate("Hello Pipecat!") - self.assertEqual(result, "Hello Pipecat!") - self.assertEqual(self.aggregator.text, "") + text = "Hello Pipecat!" + results = [agg async for agg in self.aggregator.aggregate(text)] + + # Should still be waiting for lookahead after "!" + self.assertEqual(len(results), 0) + + # Flush to get the pending sentence + result = await self.aggregator.flush() + self.assertEqual(result.text, "Hello Pipecat!") + self.assertEqual(result.type, "sentence") + self.assertEqual(self.aggregator.text.text, "") async def test_basic_tags(self): await self.aggregator.reset() # Tags involved, avoid aggregation during tags. - result = await self.aggregator.aggregate("My email is foo@pipecat.ai.") - self.assertEqual(result, "My email is foo@pipecat.ai.") - self.assertEqual(self.aggregator.text, "") + text = "My email is foo@pipecat.ai." + results = [agg async for agg in self.aggregator.aggregate(text)] + + # Should still be waiting for lookahead after "." + self.assertEqual(len(results), 0) + + # Flush to get the pending sentence + result = await self.aggregator.flush() + self.assertEqual(result.text, "My email is foo@pipecat.ai.") + self.assertEqual(result.type, "sentence") + self.assertEqual(self.aggregator.text.text, "") async def test_streaming_tags(self): await self.aggregator.reset() - # Tags involved, stream small chunk of texts. - result = await self.aggregator.aggregate("My email is foo.") - self.assertIsNone(result) - self.assertEqual(self.aggregator.text, "My email is foo.") + # Should still be waiting for lookahead after "." + self.assertEqual(len(results), 0) + self.assertEqual(self.aggregator.text.text, text) + self.assertEqual(self.aggregator.text.type, "sentence") - result = await self.aggregator.aggregate("bar@pipecat.") - self.assertIsNone(result) - self.assertEqual(self.aggregator.text, "My email is foo.bar@pipecat.") - - result = await self.aggregator.aggregate("aifoo.bar@pipecat.ai.") - self.assertEqual(result, "My email is foo.bar@pipecat.ai.") - self.assertEqual(self.aggregator.text, "") + # Flush to get the pending sentence + result = await self.aggregator.flush() + self.assertEqual(result.text, text) + self.assertEqual(self.aggregator.text.text, "") + self.assertEqual(self.aggregator.text.type, "sentence") diff --git a/tests/test_transcript_processor.py b/tests/test_transcript_processor.py index 19366086c..d86e42101 100644 --- a/tests/test_transcript_processor.py +++ b/tests/test_transcript_processor.py @@ -11,6 +11,7 @@ from datetime import datetime, timezone from typing import List, Tuple, cast from pipecat.frames.frames import ( + AggregationType, BotStartedSpeakingFrame, BotStoppedSpeakingFrame, CancelFrame, @@ -130,11 +131,11 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): frames_to_send = [ BotStartedSpeakingFrame(), SleepFrame(), # Wait for StartedSpeaking to process - TTSTextFrame(text="Hello"), - TTSTextFrame(text="world!"), - TTSTextFrame(text="How"), - TTSTextFrame(text="are"), - TTSTextFrame(text="you?"), + TTSTextFrame(text="Hello", aggregated_by=AggregationType.WORD), + TTSTextFrame(text="world!", aggregated_by=AggregationType.WORD), + TTSTextFrame(text="How", aggregated_by=AggregationType.WORD), + TTSTextFrame(text="are", aggregated_by=AggregationType.WORD), + TTSTextFrame(text="you?", aggregated_by=AggregationType.WORD), SleepFrame(), # Wait for text frames to queue BotStoppedSpeakingFrame(), ] @@ -195,9 +196,9 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): frames_to_send = [ BotStartedSpeakingFrame(), SleepFrame(), - TTSTextFrame(text=""), # Empty text - TTSTextFrame(text=" "), # Just whitespace - TTSTextFrame(text="\n"), # Just newline + TTSTextFrame(text="", aggregated_by=AggregationType.WORD), # Empty text + TTSTextFrame(text=" ", aggregated_by=AggregationType.WORD), # Just whitespace + TTSTextFrame(text="\n", aggregated_by=AggregationType.WORD), # Just newline BotStoppedSpeakingFrame(), # Pipeline ends here; run_test will automatically send EndFrame ] @@ -235,14 +236,14 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): frames_to_send = [ BotStartedSpeakingFrame(), SleepFrame(), - TTSTextFrame(text="Hello"), - TTSTextFrame(text="world!"), + TTSTextFrame(text="Hello", aggregated_by=AggregationType.WORD), + TTSTextFrame(text="world!", aggregated_by=AggregationType.WORD), SleepFrame(), InterruptionFrame(), # User interrupts here SleepFrame(), BotStartedSpeakingFrame(), - TTSTextFrame(text="New"), - TTSTextFrame(text="response"), + TTSTextFrame(text="New", aggregated_by=AggregationType.WORD), + TTSTextFrame(text="response", aggregated_by=AggregationType.WORD), SleepFrame(), BotStoppedSpeakingFrame(), ] @@ -299,8 +300,8 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): frames_to_send = [ BotStartedSpeakingFrame(), SleepFrame(), - TTSTextFrame(text="Hello"), - TTSTextFrame(text="world"), + TTSTextFrame(text="Hello", aggregated_by=AggregationType.WORD), + TTSTextFrame(text="world", aggregated_by=AggregationType.WORD), # Pipeline ends here; run_test will automatically send EndFrame ] @@ -338,8 +339,8 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): frames_to_send = [ BotStartedSpeakingFrame(), SleepFrame(), - TTSTextFrame(text="Hello"), - TTSTextFrame(text="world"), + TTSTextFrame(text="Hello", aggregated_by=AggregationType.WORD), + TTSTextFrame(text="world", aggregated_by=AggregationType.WORD), SleepFrame(), # Ensure messages are processed CancelFrame(), ] @@ -401,8 +402,8 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): frames_to_send = [ BotStartedSpeakingFrame(), SleepFrame(), - TTSTextFrame(text="Assistant"), - TTSTextFrame(text="message"), + TTSTextFrame(text="Assistant", aggregated_by=AggregationType.WORD), + TTSTextFrame(text="message", aggregated_by=AggregationType.WORD), BotStoppedSpeakingFrame(), ] @@ -439,7 +440,7 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): # Test the specific pattern shared def make_tts_text_frame(text: str) -> TTSTextFrame: - frame = TTSTextFrame(text=text) + frame = TTSTextFrame(text=text, aggregated_by=AggregationType.WORD) frame.includes_inter_frame_spaces = True return frame diff --git a/uv.lock b/uv.lock index eabe03714..cdb084284 100644 --- a/uv.lock +++ b/uv.lock @@ -36,29 +36,29 @@ wheels = [ [[package]] name = "aic-sdk" -version = "1.1.0" +version = "1.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/99/83/bf38b95d98c67b8ebc574fb4a4f23c07a3740b51992d7522976173d30b98/aic_sdk-1.1.0.tar.gz", hash = "sha256:04e08df695581c8cb4db8acca20e73815e9f449e7bd08e0162fd55518c727963", size = 34954, upload-time = "2025-11-11T20:45:24.25Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/ba/3ebe31b91e03d42437ec864e9d2af3a52b7ccc73a1a0c1026275956270b0/aic_sdk-1.2.0.tar.gz", hash = "sha256:eeda9a181c679f175dbe6f0efc0c67ec98ff3d84cfe01541fef7fa12ecd505ca", size = 35606, upload-time = "2025-11-20T14:42:14.333Z" } [[package]] name = "aioboto3" -version = "15.0.0" +version = "15.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiobotocore", extra = ["boto3"] }, { name = "aiofiles" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/80/d0/ed107e16551ba1b93ddcca9a6bf79580450945268a8bc396530687b3189f/aioboto3-15.0.0.tar.gz", hash = "sha256:dce40b701d1f8e0886dc874d27cd9799b8bf6b32d63743f57e7bef7e4a562756", size = 225278, upload-time = "2025-06-26T16:30:48.967Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/01/92e9ab00f36e2899315f49eefcd5b4685fbb19016c7f19a9edf06da80bb0/aioboto3-15.5.0.tar.gz", hash = "sha256:ea8d8787d315594842fbfcf2c4dce3bac2ad61be275bc8584b2ce9a3402a6979", size = 255069, upload-time = "2025-10-30T13:37:16.122Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bf/95/d69c744f408e5e4592fe53ed98fc244dd13b83d84cf1f83b2499d98bfcc9/aioboto3-15.0.0-py3-none-any.whl", hash = "sha256:9cf54b3627c8b34bb82eaf43ab327e7027e37f92b1e10dd5cfe343cd512568d0", size = 35785, upload-time = "2025-06-26T16:30:47.444Z" }, + { url = "https://files.pythonhosted.org/packages/e5/3e/e8f5b665bca646d43b916763c901e00a07e40f7746c9128bdc912a089424/aioboto3-15.5.0-py3-none-any.whl", hash = "sha256:cc880c4d6a8481dd7e05da89f41c384dbd841454fc1998ae25ca9c39201437a6", size = 35913, upload-time = "2025-10-30T13:37:14.549Z" }, ] [[package]] name = "aiobotocore" -version = "2.23.0" +version = "2.25.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -69,9 +69,9 @@ dependencies = [ { name = "python-dateutil" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9d/25/4b06ea1214ddf020a28df27dc7136ac9dfaf87929d51e6f6044dd350ed67/aiobotocore-2.23.0.tar.gz", hash = "sha256:0333931365a6c7053aee292fe6ef50c74690c4ae06bb019afdf706cb6f2f5e32", size = 115825, upload-time = "2025-06-12T23:46:38.055Z" } +sdist = { url = "https://files.pythonhosted.org/packages/62/94/2e4ec48cf1abb89971cb2612d86f979a6240520f0a659b53a43116d344dc/aiobotocore-2.25.1.tar.gz", hash = "sha256:ea9be739bfd7ece8864f072ec99bb9ed5c7e78ebb2b0b15f29781fbe02daedbc", size = 120560, upload-time = "2025-10-28T22:33:21.787Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ea/43/ccf9b29669cdb09fd4bfc0a8effeb2973b22a0f3c3be4142d0b485975d11/aiobotocore-2.23.0-py3-none-any.whl", hash = "sha256:8202cebbf147804a083a02bc282fbfda873bfdd0065fd34b64784acb7757b66e", size = 84161, upload-time = "2025-06-12T23:46:36.305Z" }, + { url = "https://files.pythonhosted.org/packages/95/2a/d275ec4ce5cd0096665043995a7d76f5d0524853c76a3d04656de49f8808/aiobotocore-2.25.1-py3-none-any.whl", hash = "sha256:eb6daebe3cbef5b39a0bb2a97cffbe9c7cb46b2fcc399ad141f369f3c2134b1f", size = 86039, upload-time = "2025-10-28T22:33:19.949Z" }, ] [package.optional-dependencies] @@ -419,16 +419,30 @@ wheels = [ [[package]] name = "aws-sdk-bedrock-runtime" -version = "0.1.1" +version = "0.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "smithy-aws-core", extra = ["eventstream", "json"], marker = "python_full_version >= '3.12'" }, { name = "smithy-core", marker = "python_full_version >= '3.12'" }, { name = "smithy-http", extra = ["awscrt"], marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1d/78/48574454b3cac869df67665e4a403ebfc3abfcfba2c2ff01ccfd67d55f8f/aws_sdk_bedrock_runtime-0.1.1.tar.gz", hash = "sha256:c896f99e675c3a1ab600633a07b785f3dc9fe8ab94f640b1f992b63da2dfc784", size = 82446, upload-time = "2025-10-21T20:25:25.845Z" } +sdist = { url = "https://files.pythonhosted.org/packages/db/94/f2451bb09c106e5690bbb88fc366637cdcec942b352ed9bb788804c877e0/aws_sdk_bedrock_runtime-0.2.0.tar.gz", hash = "sha256:8de52dd4492e74c73244d4b41a52304e1db368814a10e49dbbf8f4e8e412cd0e", size = 88156, upload-time = "2025-11-22T00:35:44.978Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/83/07/62c0b70223d178c138f29124ac2f7973a6ba803abc7735b6a01a85217f3d/aws_sdk_bedrock_runtime-0.1.1-py3-none-any.whl", hash = "sha256:c0336b377b2112cf88197d3d44302fbeb3efb1101989fa49ae55e78f49cfe345", size = 74954, upload-time = "2025-10-21T20:25:24.973Z" }, + { url = "https://files.pythonhosted.org/packages/eb/6b/07fbddd31dd6e38c967fe088b5e91a7cc3a2bc0f645f18b4e5d45bc03f1f/aws_sdk_bedrock_runtime-0.2.0-py3-none-any.whl", hash = "sha256:19594de50a52d199d73efca153c0a2328bd781827715a6e012d50b11085236cc", size = 79875, upload-time = "2025-11-22T00:35:44.092Z" }, +] + +[[package]] +name = "aws-sdk-sagemaker-runtime-http2" +version = "0.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "smithy-aws-core", extra = ["eventstream", "json"], marker = "python_full_version >= '3.12'" }, + { name = "smithy-core", marker = "python_full_version >= '3.12'" }, + { name = "smithy-http", extra = ["awscrt"], marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6e/ca/00f9c55887fc0f3fa345995dd871d40ff81473ab1591e56b4b4483d99d00/aws_sdk_sagemaker_runtime_http2-0.1.0.tar.gz", hash = "sha256:5077ec0c4440495b15004bbf04e27bc0bc137f1f8950d32195c6b45d7788d837", size = 20863, upload-time = "2025-11-22T00:20:56.358Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/24/2e2f727c51c20f4625cd19364d9421dbd7c893fe2b53a46eb0caaf6263a2/aws_sdk_sagemaker_runtime_http2-0.1.0-py3-none-any.whl", hash = "sha256:1aebb728ba6c6d14e58e29ecf89b51f7abbe8786d34144f8a7d59a419e80bd2f", size = 21911, upload-time = "2025-11-22T00:20:55.054Z" }, ] [[package]] @@ -606,30 +620,30 @@ wheels = [ [[package]] name = "boto3" -version = "1.38.27" +version = "1.40.61" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, { name = "jmespath" }, { name = "s3transfer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e7/96/fc74d8521d2369dd8c412438401ff12e1350a1cd3eab5c758ed3dd5e5f82/boto3-1.38.27.tar.gz", hash = "sha256:94bd7fdd92d5701b362d4df100d21e28f8307a67ff56b6a8b0398119cf22f859", size = 111875, upload-time = "2025-05-30T19:32:41.352Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ed/f9/6ef8feb52c3cce5ec3967a535a6114b57ac7949fd166b0f3090c2b06e4e5/boto3-1.40.61.tar.gz", hash = "sha256:d6c56277251adf6c2bdd25249feae625abe4966831676689ff23b4694dea5b12", size = 111535, upload-time = "2025-10-28T19:26:57.247Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/43/8b/b2361188bd1e293eede1bc165e2461d390394f71ec0c8c21211c8dabf62c/boto3-1.38.27-py3-none-any.whl", hash = "sha256:95f5fe688795303a8a15e8b7e7f255cadab35eae459d00cc281a4fd77252ea80", size = 139938, upload-time = "2025-05-30T19:32:38.006Z" }, + { url = "https://files.pythonhosted.org/packages/61/24/3bf865b07d15fea85b63504856e137029b6acbc73762496064219cdb265d/boto3-1.40.61-py3-none-any.whl", hash = "sha256:6b9c57b2a922b5d8c17766e29ed792586a818098efe84def27c8f582b33f898c", size = 139321, upload-time = "2025-10-28T19:26:55.007Z" }, ] [[package]] name = "botocore" -version = "1.38.27" +version = "1.40.61" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jmespath" }, { name = "python-dateutil" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/36/5e/67899214ad57f7f26af5bd776ac5eb583dc4ecf5c1e52e2cbfdc200e487a/botocore-1.38.27.tar.gz", hash = "sha256:9788f7efe974328a38cbade64cc0b1e67d27944b899f88cb786ae362973133b6", size = 13919963, upload-time = "2025-05-30T19:32:29.657Z" } +sdist = { url = "https://files.pythonhosted.org/packages/28/a3/81d3a47c2dbfd76f185d3b894f2ad01a75096c006a2dd91f237dca182188/botocore-1.40.61.tar.gz", hash = "sha256:a2487ad69b090f9cccd64cf07c7021cd80ee9c0655ad974f87045b02f3ef52cd", size = 14393956, upload-time = "2025-10-28T19:26:46.108Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/83/a753562020b69fa90cebc39e8af2c753b24dcdc74bee8355ee3f6cefdf34/botocore-1.38.27-py3-none-any.whl", hash = "sha256:a785d5e9a5eda88ad6ab9ed8b87d1f2ac409d0226bba6ff801c55359e94d91a8", size = 13580545, upload-time = "2025-05-30T19:32:26.712Z" }, + { url = "https://files.pythonhosted.org/packages/38/c5/f6ce561004db45f0b847c2cd9b19c67c6bf348a82018a48cb718be6b58b0/botocore-1.40.61-py3-none-any.whl", hash = "sha256:17ebae412692fd4824f99cde0f08d50126dc97954008e5ba2b522eb049238aa7", size = 14055973, upload-time = "2025-10-28T19:26:42.15Z" }, ] [[package]] @@ -1291,13 +1305,13 @@ wheels = [ [[package]] name = "daily-python" -version = "0.21.0" +version = "0.22.0" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ff/11/99590f8b7aad077f3f9b5b59d39b010aee0bd01b14dece8ae1e93d8080e7/daily_python-0.21.0-cp37-abi3-macosx_10_15_x86_64.whl", hash = "sha256:bdec96417825181559769bb2258ae688d1215949a1878336194e36fb452274a8", size = 13277066, upload-time = "2025-10-29T00:20:49.523Z" }, - { url = "https://files.pythonhosted.org/packages/e5/db/8c57f1a1b713ba3393584ac2be32d8074d3022a2c2c17c28eb4cd2aa3629/daily_python-0.21.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:18677fa1415a0dc48b891cdf2fb8fe9dabc70e1b019d5aaa3d0699ccc8d187c9", size = 11644908, upload-time = "2025-10-29T00:20:52.106Z" }, - { url = "https://files.pythonhosted.org/packages/64/b6/b03f2f58a367d6ef4bb728715471542fdfa68afa8a177670139c3a2aadb7/daily_python-0.21.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:97eb97352fe74227061b678e330b8befcfa4c694feb6eb2b09fe6eacec00ad6d", size = 13652356, upload-time = "2025-10-29T00:20:54.813Z" }, - { url = "https://files.pythonhosted.org/packages/f6/76/bde65f6f8d4c1679dc6c185fa37dae9223f6ddb4b7ced728ef46504956f7/daily_python-0.21.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:68c3e36f609fc2fce79e4d17ecf1021eadd836506db6c5125f95c682bcf3612a", size = 14304643, upload-time = "2025-10-29T00:20:57.194Z" }, + { url = "https://files.pythonhosted.org/packages/04/db/d6c311ba760123a9987a9ae291171c9a6af11ee4dbefdb661d65e2ac13a2/daily_python-0.22.0-cp37-abi3-macosx_10_15_x86_64.whl", hash = "sha256:2ef7591a7929c5d9e5ea78329ea049d2f313bd3d2d289f5f4ecce4bb3799c3d0", size = 13526264, upload-time = "2025-11-20T05:52:04.134Z" }, + { url = "https://files.pythonhosted.org/packages/ab/47/f1f6d893e7aab4b2e3d3b20d0dd8fbf31c7a71597d274aae1d288e36fac3/daily_python-0.22.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:0fbc8467c471ce536dc6214a24cc28cbc38ff61113b1714e09d0eafc2741fc5a", size = 12041400, upload-time = "2025-11-20T05:52:06.419Z" }, + { url = "https://files.pythonhosted.org/packages/9d/c9/8f26944cd55ece2ab9c076fae5c1fcf4fdc8639ea6f2b861566d26ad9e00/daily_python-0.22.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:b8aa9531b0fa2852b41935697d184be86318eaee3b35f49e0b5714e53cdb524a", size = 14147194, upload-time = "2025-11-20T05:52:08.4Z" }, + { url = "https://files.pythonhosted.org/packages/a8/02/d86f4cee39bcdb112d83e2cf12345d7a974cbad5dafb350788148644f16b/daily_python-0.22.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:8f49fceee92830aaf53a65053f332504b2cc62af79c38edb9bce8707c9fa4b0c", size = 14654605, upload-time = "2025-11-20T05:52:10.639Z" }, ] [[package]] @@ -4461,6 +4475,7 @@ daily = [ ] deepgram = [ { name = "deepgram-sdk" }, + { name = "websockets" }, ] elevenlabs = [ { name = "websockets" }, @@ -4481,6 +4496,9 @@ google = [ { name = "google-genai" }, { name = "websockets" }, ] +gradium = [ + { name = "websockets" }, +] groq = [ { name = "groq" }, ] @@ -4508,6 +4526,7 @@ langchain = [ livekit = [ { name = "livekit" }, { name = "livekit-api" }, + { name = "pyjwt" }, { name = "tenacity" }, ] lmnt = [ @@ -4548,6 +4567,9 @@ neuphonic = [ noisereduce = [ { name = "noisereduce" }, ] +nvidia = [ + { name = "nvidia-riva-client" }, +] openai = [ { name = "websockets" }, ] @@ -4569,6 +4591,9 @@ runner = [ { name = "python-dotenv" }, { name = "uvicorn" }, ] +sagemaker = [ + { name = "aws-sdk-sagemaker-runtime-http2", marker = "python_full_version >= '3.12'" }, +] sarvam = [ { name = "sarvamai" }, { name = "websockets" }, @@ -4633,6 +4658,7 @@ dev = [ { name = "ruff" }, { name = "setuptools" }, { name = "setuptools-scm" }, + { name = "towncrier" }, ] docs = [ { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -4647,18 +4673,19 @@ docs = [ [package.metadata] requires-dist = [ { name = "accelerate", marker = "extra == 'moondream'", specifier = "~=1.10.0" }, - { name = "aic-sdk", marker = "extra == 'aic'", specifier = "~=1.1.0" }, - { name = "aioboto3", marker = "extra == 'aws'", specifier = "~=15.0.0" }, + { name = "aic-sdk", marker = "extra == 'aic'", specifier = "~=1.2.0" }, + { name = "aioboto3", marker = "extra == 'aws'", specifier = "~=15.5.0" }, { name = "aiofiles", specifier = ">=24.1.0,<25" }, { name = "aiohttp", specifier = ">=3.11.12,<4" }, { name = "aiortc", marker = "extra == 'webrtc'", specifier = ">=1.13.0,<2" }, { name = "anthropic", marker = "extra == 'anthropic'", specifier = "~=0.49.0" }, { name = "audioop-lts", marker = "python_full_version >= '3.13'", specifier = "~=0.2.1" }, - { name = "aws-sdk-bedrock-runtime", marker = "python_full_version >= '3.12' and extra == 'aws-nova-sonic'", specifier = "~=0.1.1" }, + { name = "aws-sdk-bedrock-runtime", marker = "python_full_version >= '3.12' and extra == 'aws-nova-sonic'", specifier = "~=0.2.0" }, + { name = "aws-sdk-sagemaker-runtime-http2", marker = "python_full_version >= '3.12' and extra == 'sagemaker'" }, { name = "azure-cognitiveservices-speech", marker = "extra == 'azure'", specifier = "~=1.42.0" }, { name = "cartesia", marker = "extra == 'cartesia'", specifier = "~=2.0.3" }, { name = "coremltools", marker = "extra == 'local-smart-turn'", specifier = ">=8.0" }, - { name = "daily-python", marker = "extra == 'daily'", specifier = "~=0.21.0" }, + { name = "daily-python", marker = "extra == 'daily'", specifier = "~=0.22.0" }, { name = "deepgram-sdk", marker = "extra == 'deepgram'", specifier = "~=4.7.0" }, { name = "docstring-parser", specifier = "~=0.16" }, { name = "einops", marker = "extra == 'moondream'", specifier = "~=0.8.0" }, @@ -4686,7 +4713,7 @@ requires-dist = [ { name = "noisereduce", marker = "extra == 'noisereduce'", specifier = "~=3.0.3" }, { name = "numba", specifier = "==0.61.2" }, { name = "numpy", specifier = ">=1.26.4,<3" }, - { name = "nvidia-riva-client", marker = "extra == 'riva'", specifier = "~=2.21.1" }, + { name = "nvidia-riva-client", marker = "extra == 'nvidia'", specifier = "~=2.21.1" }, { name = "onnxruntime", marker = "extra == 'local-smart-turn-v3'", specifier = ">=1.20.1,<2" }, { name = "onnxruntime", marker = "extra == 'silero'", specifier = ">=1.20.1,<2" }, { name = "openai", specifier = ">=1.74.0,<3" }, @@ -4697,14 +4724,17 @@ requires-dist = [ { name = "opentelemetry-sdk", marker = "extra == 'tracing'", specifier = ">=1.33.0" }, { name = "ormsgpack", marker = "extra == 'fish'", specifier = "~=1.7.0" }, { name = "pillow", specifier = ">=11.1.0,<12" }, + { name = "pipecat-ai", extras = ["nvidia"], marker = "extra == 'riva'" }, { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'assemblyai'" }, { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'asyncai'" }, { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'aws'" }, { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'cartesia'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'deepgram'" }, { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'elevenlabs'" }, { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'fish'" }, { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'gladia'" }, { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'google'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'gradium'" }, { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'heygen'" }, { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'lmnt'" }, { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'neuphonic'" }, @@ -4721,13 +4751,14 @@ requires-dist = [ { name = "pyaudio", marker = "extra == 'local'", specifier = "~=0.2.14" }, { name = "pydantic", specifier = ">=2.10.6,<3" }, { name = "pygobject", marker = "extra == 'gstreamer'", specifier = "~=3.50.0" }, + { name = "pyjwt", marker = "extra == 'livekit'", specifier = ">=2.10.1" }, { name = "pyloudnorm", specifier = "~=0.1.1" }, { name = "python-dotenv", marker = "extra == 'runner'", specifier = ">=1.0.0,<2.0.0" }, { name = "pyvips", extras = ["binary"], marker = "extra == 'moondream'", specifier = "~=3.0.0" }, { name = "resampy", specifier = "~=0.4.3" }, { name = "sarvamai", marker = "extra == 'sarvam'", specifier = "==0.1.21" }, { name = "sentry-sdk", marker = "extra == 'sentry'", specifier = ">=2.28.0,<3" }, - { name = "simli-ai", marker = "extra == 'simli'", specifier = "~=0.1.25" }, + { name = "simli-ai", marker = "extra == 'simli'", specifier = "~=1.0.3" }, { name = "soundfile", marker = "extra == 'soundfile'", specifier = "~=0.13.1" }, { name = "soxr", specifier = "~=0.5.0" }, { name = "speechmatics-rt", marker = "extra == 'speechmatics'", specifier = ">=0.5.0" }, @@ -4745,7 +4776,7 @@ requires-dist = [ { name = "wait-for2", marker = "python_full_version < '3.12'", specifier = ">=0.4.1" }, { name = "websockets", marker = "extra == 'websockets-base'", specifier = ">=13.1,<16.0" }, ] -provides-extras = ["aic", "anthropic", "assemblyai", "asyncai", "aws", "aws-nova-sonic", "azure", "cartesia", "cerebras", "deepseek", "daily", "deepgram", "elevenlabs", "fal", "fireworks", "fish", "gladia", "google", "grok", "groq", "gstreamer", "heygen", "hume", "inworld", "krisp", "koala", "langchain", "livekit", "lmnt", "local", "mcp", "mem0", "mistral", "mlx-whisper", "moondream", "nim", "neuphonic", "noisereduce", "openai", "openpipe", "openrouter", "perplexity", "playht", "qwen", "rime", "riva", "runner", "sambanova", "sarvam", "sentry", "local-smart-turn", "local-smart-turn-v3", "remote-smart-turn", "silero", "simli", "soniox", "soundfile", "speechmatics", "strands", "tavus", "together", "tracing", "ultravox", "webrtc", "websocket", "websockets-base", "whisper"] +provides-extras = ["aic", "anthropic", "assemblyai", "asyncai", "aws", "aws-nova-sonic", "azure", "cartesia", "cerebras", "daily", "deepgram", "deepseek", "elevenlabs", "fal", "fireworks", "fish", "gladia", "google", "gradium", "grok", "groq", "gstreamer", "heygen", "hume", "inworld", "koala", "krisp", "langchain", "livekit", "lmnt", "local", "local-smart-turn", "local-smart-turn-v3", "mcp", "mem0", "mistral", "mlx-whisper", "moondream", "neuphonic", "noisereduce", "nvidia", "openai", "openpipe", "openrouter", "perplexity", "playht", "qwen", "remote-smart-turn", "rime", "riva", "runner", "sagemaker", "sambanova", "sarvam", "sentry", "silero", "simli", "soniox", "soundfile", "speechmatics", "strands", "tavus", "together", "tracing", "ultravox", "webrtc", "websocket", "websockets-base", "whisper"] [package.metadata.requires-dev] dev = [ @@ -4762,6 +4793,7 @@ dev = [ { name = "ruff", specifier = ">=0.12.11,<1" }, { name = "setuptools", specifier = "~=78.1.1" }, { name = "setuptools-scm", specifier = "~=8.3.1" }, + { name = "towncrier", specifier = "~=25.8.0" }, ] docs = [ { name = "sphinx", specifier = ">=8.1.3" }, @@ -6202,14 +6234,14 @@ wheels = [ [[package]] name = "s3transfer" -version = "0.13.1" +version = "0.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6d/05/d52bf1e65044b4e5e27d4e63e8d1579dbdec54fce685908ae09bc3720030/s3transfer-0.13.1.tar.gz", hash = "sha256:c3fdba22ba1bd367922f27ec8032d6a1cf5f10c934fb5d68cf60fd5a23d936cf", size = 150589, upload-time = "2025-07-18T19:22:42.31Z" } +sdist = { url = "https://files.pythonhosted.org/packages/62/74/8d69dcb7a9efe8baa2046891735e5dfe433ad558ae23d9e3c14c633d1d58/s3transfer-0.14.0.tar.gz", hash = "sha256:eff12264e7c8b4985074ccce27a3b38a485bb7f7422cc8046fee9be4983e4125", size = 151547, upload-time = "2025-09-09T19:23:31.089Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6d/4f/d073e09df851cfa251ef7840007d04db3293a0482ce607d2b993926089be/s3transfer-0.13.1-py3-none-any.whl", hash = "sha256:a981aa7429be23fe6dfc13e80e4020057cbab622b08c0315288758d67cabc724", size = 85308, upload-time = "2025-07-18T19:22:40.947Z" }, + { url = "https://files.pythonhosted.org/packages/48/f0/ae7ca09223a81a1d890b2557186ea015f6e0502e9b8cb8e1813f1d8cfa4e/s3transfer-0.14.0-py3-none-any.whl", hash = "sha256:ea3b790c7077558ed1f02a3072fb3cb992bbbd253392f4b6e9e8976941c7d456", size = 85712, upload-time = "2025-09-09T19:23:30.041Z" }, ] [[package]] @@ -6496,18 +6528,19 @@ wheels = [ [[package]] name = "simli-ai" -version = "0.1.25" +version = "1.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiortc" }, { name = "av" }, { name = "httpx" }, + { name = "livekit" }, { name = "numpy" }, { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/64/6a/b28f90baf76f6a60865985f6233ff44abc72d45b66b76658bff3961e20a7/simli_ai-0.1.25.tar.gz", hash = "sha256:7a00b3426dc26a6a421641072c3e49014b7950c621cf4544152f35c58d13fcff", size = 13182, upload-time = "2025-11-06T16:27:08.862Z" } +sdist = { url = "https://files.pythonhosted.org/packages/81/03/b0b3e12c68fd3f9c57f6afeee67841349e4866b88760f413357af3043ae4/simli_ai-1.0.3.tar.gz", hash = "sha256:e96b0621a1dbd9582b2ae3d51eefd4995983b49c1f1061eb9239707b15a1ee27", size = 13350, upload-time = "2025-11-13T12:22:32.514Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ac/57/ae1032fd88214ea4ee6d3028c817c12a999eb90a67766bbab31e9819385a/simli_ai-0.1.25-py3-none-any.whl", hash = "sha256:7d01f65321dc9052f25e15d0463af6a20a86c6d37d9a7b3a2c4b01cbec0a54ed", size = 13651, upload-time = "2025-11-06T16:27:07.765Z" }, + { url = "https://files.pythonhosted.org/packages/5e/d1/dc382ba529de0d2d51f35e9bfd20b41d8f5c96404a3aa24bae97a5a5e51f/simli_ai-1.0.3-py3-none-any.whl", hash = "sha256:ffafa7540aa28833e207be8f3b199367c7f500dac1a8ba0108395bfb7d8362bc", size = 13863, upload-time = "2025-11-13T12:22:31.218Z" }, ] [[package]] @@ -6521,16 +6554,16 @@ wheels = [ [[package]] name = "smithy-aws-core" -version = "0.1.1" +version = "0.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aws-sdk-signers", marker = "python_full_version >= '3.12'" }, { name = "smithy-core", marker = "python_full_version >= '3.12'" }, { name = "smithy-http", marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/56/d3/f847e0fd36b95aa36ce3a4c9ce1a08e16b2aa9a56b71714045c9c924e282/smithy_aws_core-0.1.1.tar.gz", hash = "sha256:78dfd7040fc2bc72b6af293096642fc9a7bfd2db28ddbdf7c4110535eab9d662", size = 11196, upload-time = "2025-10-21T20:21:18.648Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c1/c8/5970c869527972b23a1733de3993d54283c84a2340e84acdd48a11aa0ff4/smithy_aws_core-0.2.0.tar.gz", hash = "sha256:dfa1ecd311d6f0a16f48c86d793085e2a0a33a46de897d129dd1f142a43bf7f6", size = 11344, upload-time = "2025-11-21T18:33:01.928Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d0/04/87cb06f0f6d664b5cffdef6d4042dd52c11c138436084d30ffdaa3543031/smithy_aws_core-0.1.1-py3-none-any.whl", hash = "sha256:0d1634f276c2999dc2a04fafef63b9d28309de50d939d1d49df952773a7063c4", size = 18963, upload-time = "2025-10-21T20:21:17.692Z" }, + { url = "https://files.pythonhosted.org/packages/88/25/739c0005a6cb4effbc2d37fe23590660b508fe314200f4acf94410a8f315/smithy_aws_core-0.2.0-py3-none-any.whl", hash = "sha256:d112082ef77758e1977f8694cf690ac35c76570c12a6590fccd5da085a3ce507", size = 18966, upload-time = "2025-11-21T18:33:00.812Z" }, ] [package.optional-dependencies] @@ -6543,35 +6576,35 @@ json = [ [[package]] name = "smithy-aws-event-stream" -version = "0.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "smithy-core", marker = "python_full_version >= '3.12'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/49/26/8ff24194efed60b2df18f610ea05fa2a4c6546858b80a0a51335a4943b9b/smithy_aws_event_stream-0.1.0.tar.gz", hash = "sha256:6634691a3bf5d4801a2c29f0761db2dc4771f3ae43cdee50c10d4b4bb2f86475", size = 12216, upload-time = "2025-09-29T19:37:14.659Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/90/c4/2b63d31af58fc359577e5515bf730348a235f2f2fa10e17af8640495c81c/smithy_aws_event_stream-0.1.0-py3-none-any.whl", hash = "sha256:17a7300a85cb90df4c6c23f895ca6343361fa419203c3cf80019edd7d3b5f036", size = 15581, upload-time = "2025-09-29T19:37:13.589Z" }, -] - -[[package]] -name = "smithy-core" -version = "0.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b9/8d/16028d03456071d21de7591f1e1e6a1cc81b2389e53ef8663dbf59caf9cd/smithy_core-0.1.0.tar.gz", hash = "sha256:b159b8905264e1e4c613eab9f74cec0b2f5b8119c42fbadddb4da0a8ed8050e9", size = 48415, upload-time = "2025-09-29T19:37:16.873Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ca/5b/563cb2beadcfa40597b0c3ff3f2d42e21f065b14782c4ba9cb41a44b745f/smithy_core-0.1.0-py3-none-any.whl", hash = "sha256:cb44e9355fb89e89f2c6ba6a1d59c5db4f2f7282c72d31d9307b6202d66cd0fa", size = 62895, upload-time = "2025-09-29T19:37:15.917Z" }, -] - -[[package]] -name = "smithy-http" version = "0.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "smithy-core", marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3c/1c/44e99a7dfb8c39bf0c3d998accdf4573a7a3488863b90f21af260cec2d45/smithy_http-0.2.0.tar.gz", hash = "sha256:2382562fa9af326be455f14b18615f16ffe9db756e51b2a4ca0d23e1b881cff8", size = 26729, upload-time = "2025-10-21T20:21:06.146Z" } +sdist = { url = "https://files.pythonhosted.org/packages/31/90/78283c21484f8cf9862982e53bc2769b784910735fb5fb2400a17bfb5fdd/smithy_aws_event_stream-0.2.0.tar.gz", hash = "sha256:99700a11346e7ab1435ff2e53e6f6d60a1e857f2b2ee1941d40b54270adf3323", size = 12278, upload-time = "2025-11-21T18:33:03.79Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/e2/d475fad81ac74ec0e145cb6d72afe5ecde4e2358bd632c2fd5d3f4bc87dc/smithy_http-0.2.0-py3-none-any.whl", hash = "sha256:49ee2402d7737798d70f99f491fbfb2a5767283ae562e21b6f86e3fd14f3e3e0", size = 37328, upload-time = "2025-10-21T20:21:05.362Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f5/08b997eee81b55150496ce565f0e03c72d0c80e5b218170bdeae7c46a5a4/smithy_aws_event_stream-0.2.0-py3-none-any.whl", hash = "sha256:679a0c7d944e67d3a55d287541b3ca1e61f9d6a62e13401367dcc034e75aa55d", size = 15567, upload-time = "2025-11-21T18:33:02.711Z" }, +] + +[[package]] +name = "smithy-core" +version = "0.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/f6/140f0be9331dd7cd8fa012b3ca4735df39a1a81d03eea89728f997249116/smithy_core-0.2.0.tar.gz", hash = "sha256:05c3e3309df5dcb9cf53e241bd57a96510e4575186443ea157db9dbb59b6c85e", size = 50334, upload-time = "2025-11-21T18:33:05.697Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/e3/d0defa2acf50b91625fe15e3ddb0c8e41ff64363a1f4cd9b8f19ae2ec0c6/smithy_core-0.2.0-py3-none-any.whl", hash = "sha256:db4620da3497abb60f79ac1d8a738d3eac46d7e820bfb50c777c36e932915239", size = 64777, upload-time = "2025-11-21T18:33:04.591Z" }, +] + +[[package]] +name = "smithy-http" +version = "0.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "smithy-core", marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/c7/4d8be56e897f99f3b6ffcdf52ba00a468febc939fca85b90f1c122450830/smithy_http-0.3.0.tar.gz", hash = "sha256:55dcc3af315eee6863d2f3f58ada1d9cb4bcc3a57faac10a1b21d4a93722f520", size = 28674, upload-time = "2025-11-21T18:33:07.387Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/e5/59ae79ecdc9a935ad10512c581b3054ebb1afd90498ecc8afaf141dbc22b/smithy_http-0.3.0-py3-none-any.whl", hash = "sha256:972924304febd77c7134a7cffab83ce3b48423ff966dcc1f257e2c0d58fa9b18", size = 40520, upload-time = "2025-11-21T18:33:06.312Z" }, ] [package.optional-dependencies] @@ -6581,15 +6614,15 @@ awscrt = [ [[package]] name = "smithy-json" -version = "0.1.0" +version = "0.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "ijson", marker = "python_full_version >= '3.12'" }, { name = "smithy-core", marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e2/5b/0ecb10007475e1b8faca3bbff1be2fc6edb3ea12ffc5e939e2249be95325/smithy_json-0.1.0.tar.gz", hash = "sha256:84fb48e445b87d850c240d837702c16b259ea53bad76b655ac1bbd8094d48912", size = 7086, upload-time = "2025-09-29T19:37:20.432Z" } +sdist = { url = "https://files.pythonhosted.org/packages/89/cf/e319a2a299b27bc0addf46ee3d4b9c25ec0817e3a0507b2b7a33eddc19f1/smithy_json-0.2.0.tar.gz", hash = "sha256:0946066fdda15d6a579dfdd4b61a547ab915eb057bd176fc2bc17d01dc789499", size = 7157, upload-time = "2025-11-21T18:33:08.968Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/62/95/e11c04e56aae12b62e38c49000004a1dc598a64dc207018c08448efde322/smithy_json-0.1.0-py3-none-any.whl", hash = "sha256:80ff64734dccdabf1ba6a2908555b97e60f62c07c3a27df48e421ee058413cb9", size = 9914, upload-time = "2025-09-29T19:37:19.459Z" }, + { url = "https://files.pythonhosted.org/packages/2e/b1/33012ac5b2e5940a00b6e1ccc313330e6f8692152a151f72a398cd6be0e0/smithy_json-0.2.0-py3-none-any.whl", hash = "sha256:5018a4e61731afa3094a02d737d4f956dbf270c271410c089045a17d86fc3b3b", size = 9911, upload-time = "2025-11-21T18:33:08.267Z" }, ] [[package]] @@ -7220,6 +7253,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/7b/30d423bdb2546250d719d7821aaf9058cc093d165565b245b159c788a9dd/torchvision-0.22.0-cp313-cp313t-win_amd64.whl", hash = "sha256:e5d680162694fac4c8a374954e261ddfb4eb0ce103287b0f693e4e9c579ef957", size = 1638621, upload-time = "2025-04-23T14:41:46.06Z" }, ] +[[package]] +name = "towncrier" +version = "25.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "jinja2" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c2/eb/5bf25a34123698d3bbab39c5bc5375f8f8bcbcc5a136964ade66935b8b9d/towncrier-25.8.0.tar.gz", hash = "sha256:eef16d29f831ad57abb3ae32a0565739866219f1ebfbdd297d32894eb9940eb1", size = 76322, upload-time = "2025-08-30T11:41:55.393Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/06/8ba22ec32c74ac1be3baa26116e3c28bc0e76a5387476921d20b6fdade11/towncrier-25.8.0-py3-none-any.whl", hash = "sha256:b953d133d98f9aeae9084b56a3563fd2519dfc6ec33f61c9cd2c61ff243fb513", size = 65101, upload-time = "2025-08-30T11:41:53.644Z" }, +] + [[package]] name = "tqdm" version = "4.67.1"