diff --git a/CHANGELOG.md b/CHANGELOG.md index 079142b9a..21abadaee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,14 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.0.66] - 2025-05-02 ### Added -- Added support for cross-platform local smart turn detection. You can use +- Added two new input parameters to `RimeTTSService`: `pause_between_brackets` + and `phonemize_between_brackets`. + +- Added support for cross-platform local smart turn detection. You can use `LocalSmartTurnAnalyzer` for on-device inference using Torch. - `BaseOutputTransport` now allows multiple destinations if the transport @@ -116,6 +119,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 case there's no need to push audio to the rest of the pipeline, but this is not a very common case. +- Added `RivaSegmentedSTTService`, which allows Riva offline/batch models, such + as to be "canary-1b-asr" used in Pipecat. + ### Deprecated - Function calls with parameters @@ -131,8 +137,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `TransportParams.vad_audio_passthrough` parameter is now deprecated, use `TransportParams.audio_in_passthrough` instead. +- `ParakeetSTTService` is now deprecated, use `RivaSTTService` instead, which uses + the model "parakeet-ctc-1.1b-asr" by default. + +- `FastPitchTTSService` is now deprecated, use `RivaTTSService` instead, which uses + the model "magpie-tts-multilingual" by default. + ### Fixed +- Fixed an issue with `SimliVideoService` where the bot was continuously outputting + audio, which prevents the `BotStoppedSpeakingFrame` from being emitted. + - Fixed an issue where `OpenAIRealtimeBetaLLMService` would add two assistant messages to the context. diff --git a/examples/foundational/07r-interruptible-riva-nim.py b/examples/foundational/07r-interruptible-riva-nim.py index 915beda51..ddb80181c 100644 --- a/examples/foundational/07r-interruptible-riva-nim.py +++ b/examples/foundational/07r-interruptible-riva-nim.py @@ -16,8 +16,12 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.nim.llm import NimLLMService -from pipecat.services.riva.stt import ParakeetSTTService -from pipecat.services.riva.tts import FastPitchTTSService +from pipecat.services.riva.stt import ( + ParakeetSTTService, + RivaSegmentedSTTService, + RivaSTTService, +) +from pipecat.services.riva.tts import FastPitchTTSService, RivaTTSService from pipecat.transports.base_transport import TransportParams from pipecat.transports.network.small_webrtc import SmallWebRTCTransport from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection @@ -37,11 +41,11 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac ), ) - stt = ParakeetSTTService(api_key=os.getenv("NVIDIA_API_KEY")) + stt = RivaSTTService(api_key=os.getenv("NVIDIA_API_KEY")) llm = NimLLMService(api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.1-405b-instruct") - tts = FastPitchTTSService(api_key=os.getenv("NVIDIA_API_KEY")) + tts = RivaTTSService(api_key=os.getenv("NVIDIA_API_KEY")) messages = [ { diff --git a/examples/foundational/27-simli-layer.py b/examples/foundational/27-simli-layer.py index 513bbc239..38721e50e 100644 --- a/examples/foundational/27-simli-layer.py +++ b/examples/foundational/27-simli-layer.py @@ -36,6 +36,7 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac audio_in_enabled=True, audio_out_enabled=True, video_out_enabled=True, + video_out_is_live=True, video_out_width=512, video_out_height=512, vad_analyzer=SileroVADAnalyzer(), diff --git a/examples/storytelling-chatbot/.dockerignore b/examples/storytelling-chatbot/.dockerignore deleted file mode 100644 index a6b5f4231..000000000 --- a/examples/storytelling-chatbot/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -frontend/node_modules -frontend/out \ No newline at end of file diff --git a/examples/storytelling-chatbot/README.md b/examples/storytelling-chatbot/README.md index 08da91044..d10620eef 100644 --- a/examples/storytelling-chatbot/README.md +++ b/examples/storytelling-chatbot/README.md @@ -1,4 +1,4 @@ -[![Try](https://img.shields.io/badge/try_it-here-blue)](https://storytelling-chatbot.fly.dev) +[![Try](https://img.shields.io/badge/try_it-here-blue)](https://gemini-storybot.vercel.app/) # Storytelling Chatbot @@ -9,7 +9,6 @@ It periodically prompts the user for input for a 'choose your own adventure' sty We use Gemini 2.0 for creating the story and image prompts, and we add visual elements to the story by generating images using Google's Imagen. - --- ### It uses the following AI services: @@ -20,7 +19,7 @@ Transcribes inbound participant voice media to text. **Google Gemini 2.0 - LLM** -Our creative writer LLM. You can see the context used to prompt it [here](src/prompts.py) +Our creative writer LLM. You can see the context used to prompt it [here](server/prompts.py) **ElevenLabs - Text-to-Speech** @@ -34,47 +33,76 @@ Adds pictures to our story. Prompting is quite key for style consistency, so we ## Setup -**Install requirements** +### Client -```shell -python3 -m venv venv -source venv/bin/activate -pip install -r requirements.txt -``` +1. Navigate to the client directory: -**Create environment file and set variables:** + ```shell + cd client + ``` -```shell -mv env.example .env -``` +2. Install dependencies: -When deploying to production, to ensure only this app can spawn a new bot, set your `ENV` to `production` + ```shell + npm install + ``` -**Build the frontend:** +3. Build the client: -This project uses a custom frontend, which needs to built. Note: this is done automatically as part of the Docker deployment. + ```shell + npm run build + ``` -```shell -cd frontend/ -npm install -npm run build -``` +### Server -The build UI files can be found in `frontend/out` +1. Navigate to the server directory -## Running it locally + ```shell + cd ../server + ``` -Start the API / bot manager: +2. Set up your virtual environment and install requirements -`python src/bot_runner.py --host localhost` + ```shell + python3 -m venv venv + source venv/bin/activate + pip install -r requirements.txt + ``` -If you'd like to run a custom domain or port: +3. Create environment file and set variables -`python src/bot_runner.py --host somehost --p someport` + ```shell + mv env.example .env + ``` -➡️ Open the host URL in your browser `http://localhost:7860` + You'll need API keys for: -If you've run previous versions of the demo, make sure to set `ENV=dev`, and remove the `RUN_AS_VM` line from the .env file. + - DAILY_API_KEY + - ELEVENLABS_API_KEY + - ELEVENLABS_VOICE_ID + - GOOGLE_API_KEY + +4. (Optional) Deployment: + + When deploying to production, to ensure only this app can spawn new bot processes, set your `ENV` to `production` + +## Run it locally + +1. Navigate back to the demo's root directory: + + ```shell + cd .. + ``` + +2. Run the application: + + ```shell + python server/bot_runner.py --host localhost + ``` + + You can run with a custom domain or port using: `python server/bot_runner.py --host somehost --p someport` + +3. ➡️ Open the host URL in your browser: http://localhost:7860 --- diff --git a/examples/storytelling-chatbot/frontend/.eslintrc.json b/examples/storytelling-chatbot/client/.eslintrc.json similarity index 100% rename from examples/storytelling-chatbot/frontend/.eslintrc.json rename to examples/storytelling-chatbot/client/.eslintrc.json diff --git a/examples/storytelling-chatbot/frontend/.gitignore b/examples/storytelling-chatbot/client/.gitignore similarity index 100% rename from examples/storytelling-chatbot/frontend/.gitignore rename to examples/storytelling-chatbot/client/.gitignore diff --git a/examples/storytelling-chatbot/frontend/README.md b/examples/storytelling-chatbot/client/README.md similarity index 100% rename from examples/storytelling-chatbot/frontend/README.md rename to examples/storytelling-chatbot/client/README.md diff --git a/examples/storytelling-chatbot/frontend/app/favicon.ico b/examples/storytelling-chatbot/client/app/favicon.ico similarity index 100% rename from examples/storytelling-chatbot/frontend/app/favicon.ico rename to examples/storytelling-chatbot/client/app/favicon.ico diff --git a/examples/storytelling-chatbot/frontend/app/globals.css b/examples/storytelling-chatbot/client/app/globals.css similarity index 100% rename from examples/storytelling-chatbot/frontend/app/globals.css rename to examples/storytelling-chatbot/client/app/globals.css diff --git a/examples/storytelling-chatbot/frontend/app/layout.tsx b/examples/storytelling-chatbot/client/app/layout.tsx similarity index 100% rename from examples/storytelling-chatbot/frontend/app/layout.tsx rename to examples/storytelling-chatbot/client/app/layout.tsx diff --git a/examples/storytelling-chatbot/frontend/app/opengraph-image.png b/examples/storytelling-chatbot/client/app/opengraph-image.png similarity index 100% rename from examples/storytelling-chatbot/frontend/app/opengraph-image.png rename to examples/storytelling-chatbot/client/app/opengraph-image.png diff --git a/examples/storytelling-chatbot/frontend/app/page.tsx b/examples/storytelling-chatbot/client/app/page.tsx similarity index 100% rename from examples/storytelling-chatbot/frontend/app/page.tsx rename to examples/storytelling-chatbot/client/app/page.tsx diff --git a/examples/storytelling-chatbot/frontend/app/twitter-image.png b/examples/storytelling-chatbot/client/app/twitter-image.png similarity index 100% rename from examples/storytelling-chatbot/frontend/app/twitter-image.png rename to examples/storytelling-chatbot/client/app/twitter-image.png diff --git a/examples/storytelling-chatbot/frontend/app/utils.ts b/examples/storytelling-chatbot/client/app/utils.ts similarity index 100% rename from examples/storytelling-chatbot/frontend/app/utils.ts rename to examples/storytelling-chatbot/client/app/utils.ts diff --git a/examples/storytelling-chatbot/frontend/components.json b/examples/storytelling-chatbot/client/components.json similarity index 100% rename from examples/storytelling-chatbot/frontend/components.json rename to examples/storytelling-chatbot/client/components.json diff --git a/examples/storytelling-chatbot/frontend/components/App.tsx b/examples/storytelling-chatbot/client/components/App.tsx similarity index 100% rename from examples/storytelling-chatbot/frontend/components/App.tsx rename to examples/storytelling-chatbot/client/components/App.tsx diff --git a/examples/storytelling-chatbot/frontend/components/AudioIndicator/index.tsx b/examples/storytelling-chatbot/client/components/AudioIndicator/index.tsx similarity index 100% rename from examples/storytelling-chatbot/frontend/components/AudioIndicator/index.tsx rename to examples/storytelling-chatbot/client/components/AudioIndicator/index.tsx diff --git a/examples/storytelling-chatbot/frontend/components/DevicePicker/index.tsx b/examples/storytelling-chatbot/client/components/DevicePicker/index.tsx similarity index 100% rename from examples/storytelling-chatbot/frontend/components/DevicePicker/index.tsx rename to examples/storytelling-chatbot/client/components/DevicePicker/index.tsx diff --git a/examples/storytelling-chatbot/frontend/components/MicToggle/index.tsx b/examples/storytelling-chatbot/client/components/MicToggle/index.tsx similarity index 100% rename from examples/storytelling-chatbot/frontend/components/MicToggle/index.tsx rename to examples/storytelling-chatbot/client/components/MicToggle/index.tsx diff --git a/examples/storytelling-chatbot/frontend/components/Setup.tsx b/examples/storytelling-chatbot/client/components/Setup.tsx similarity index 100% rename from examples/storytelling-chatbot/frontend/components/Setup.tsx rename to examples/storytelling-chatbot/client/components/Setup.tsx diff --git a/examples/storytelling-chatbot/frontend/components/Story.tsx b/examples/storytelling-chatbot/client/components/Story.tsx similarity index 100% rename from examples/storytelling-chatbot/frontend/components/Story.tsx rename to examples/storytelling-chatbot/client/components/Story.tsx diff --git a/examples/storytelling-chatbot/frontend/components/StoryTranscript/StoryTranscript.module.css b/examples/storytelling-chatbot/client/components/StoryTranscript/StoryTranscript.module.css similarity index 100% rename from examples/storytelling-chatbot/frontend/components/StoryTranscript/StoryTranscript.module.css rename to examples/storytelling-chatbot/client/components/StoryTranscript/StoryTranscript.module.css diff --git a/examples/storytelling-chatbot/frontend/components/StoryTranscript/index.tsx b/examples/storytelling-chatbot/client/components/StoryTranscript/index.tsx similarity index 100% rename from examples/storytelling-chatbot/frontend/components/StoryTranscript/index.tsx rename to examples/storytelling-chatbot/client/components/StoryTranscript/index.tsx diff --git a/examples/storytelling-chatbot/frontend/components/UserInputIndicator/UserInputIndicator.module.css b/examples/storytelling-chatbot/client/components/UserInputIndicator/UserInputIndicator.module.css similarity index 100% rename from examples/storytelling-chatbot/frontend/components/UserInputIndicator/UserInputIndicator.module.css rename to examples/storytelling-chatbot/client/components/UserInputIndicator/UserInputIndicator.module.css diff --git a/examples/storytelling-chatbot/frontend/components/UserInputIndicator/index.tsx b/examples/storytelling-chatbot/client/components/UserInputIndicator/index.tsx similarity index 100% rename from examples/storytelling-chatbot/frontend/components/UserInputIndicator/index.tsx rename to examples/storytelling-chatbot/client/components/UserInputIndicator/index.tsx diff --git a/examples/storytelling-chatbot/frontend/components/VideoTile/VideoTile.module.css b/examples/storytelling-chatbot/client/components/VideoTile/VideoTile.module.css similarity index 100% rename from examples/storytelling-chatbot/frontend/components/VideoTile/VideoTile.module.css rename to examples/storytelling-chatbot/client/components/VideoTile/VideoTile.module.css diff --git a/examples/storytelling-chatbot/frontend/components/VideoTile/index.tsx b/examples/storytelling-chatbot/client/components/VideoTile/index.tsx similarity index 100% rename from examples/storytelling-chatbot/frontend/components/VideoTile/index.tsx rename to examples/storytelling-chatbot/client/components/VideoTile/index.tsx diff --git a/examples/storytelling-chatbot/frontend/components/WaveText/WaveText.module.css b/examples/storytelling-chatbot/client/components/WaveText/WaveText.module.css similarity index 100% rename from examples/storytelling-chatbot/frontend/components/WaveText/WaveText.module.css rename to examples/storytelling-chatbot/client/components/WaveText/WaveText.module.css diff --git a/examples/storytelling-chatbot/frontend/components/WaveText/index.tsx b/examples/storytelling-chatbot/client/components/WaveText/index.tsx similarity index 100% rename from examples/storytelling-chatbot/frontend/components/WaveText/index.tsx rename to examples/storytelling-chatbot/client/components/WaveText/index.tsx diff --git a/examples/storytelling-chatbot/frontend/components/ui/button.tsx b/examples/storytelling-chatbot/client/components/ui/button.tsx similarity index 100% rename from examples/storytelling-chatbot/frontend/components/ui/button.tsx rename to examples/storytelling-chatbot/client/components/ui/button.tsx diff --git a/examples/storytelling-chatbot/frontend/components/ui/select.tsx b/examples/storytelling-chatbot/client/components/ui/select.tsx similarity index 100% rename from examples/storytelling-chatbot/frontend/components/ui/select.tsx rename to examples/storytelling-chatbot/client/components/ui/select.tsx diff --git a/examples/storytelling-chatbot/frontend/components/ui/typewriter.tsx b/examples/storytelling-chatbot/client/components/ui/typewriter.tsx similarity index 100% rename from examples/storytelling-chatbot/frontend/components/ui/typewriter.tsx rename to examples/storytelling-chatbot/client/components/ui/typewriter.tsx diff --git a/examples/storytelling-chatbot/frontend/env.example b/examples/storytelling-chatbot/client/env.example similarity index 100% rename from examples/storytelling-chatbot/frontend/env.example rename to examples/storytelling-chatbot/client/env.example diff --git a/examples/storytelling-chatbot/frontend/next.config.mjs b/examples/storytelling-chatbot/client/next.config.mjs similarity index 100% rename from examples/storytelling-chatbot/frontend/next.config.mjs rename to examples/storytelling-chatbot/client/next.config.mjs diff --git a/examples/storytelling-chatbot/frontend/package-lock.json b/examples/storytelling-chatbot/client/package-lock.json similarity index 99% rename from examples/storytelling-chatbot/frontend/package-lock.json rename to examples/storytelling-chatbot/client/package-lock.json index f94e74f85..57761664d 100644 --- a/examples/storytelling-chatbot/frontend/package-lock.json +++ b/examples/storytelling-chatbot/client/package-lock.json @@ -1,11 +1,11 @@ { - "name": "frontend", + "name": "client", "version": "0.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "frontend", + "name": "client", "version": "0.1.0", "dependencies": { "@daily-co/daily-js": "^0.62.0", diff --git a/examples/storytelling-chatbot/frontend/package.json b/examples/storytelling-chatbot/client/package.json similarity index 97% rename from examples/storytelling-chatbot/frontend/package.json rename to examples/storytelling-chatbot/client/package.json index b68568b4f..4b34d1059 100644 --- a/examples/storytelling-chatbot/frontend/package.json +++ b/examples/storytelling-chatbot/client/package.json @@ -1,5 +1,5 @@ { - "name": "frontend", + "name": "client", "version": "0.1.0", "private": true, "scripts": { diff --git a/examples/storytelling-chatbot/frontend/postcss.config.js b/examples/storytelling-chatbot/client/postcss.config.js similarity index 100% rename from examples/storytelling-chatbot/frontend/postcss.config.js rename to examples/storytelling-chatbot/client/postcss.config.js diff --git a/examples/storytelling-chatbot/frontend/public/alpha-mask.gif b/examples/storytelling-chatbot/client/public/alpha-mask.gif similarity index 100% rename from examples/storytelling-chatbot/frontend/public/alpha-mask.gif rename to examples/storytelling-chatbot/client/public/alpha-mask.gif diff --git a/examples/storytelling-chatbot/frontend/public/bg.jpg b/examples/storytelling-chatbot/client/public/bg.jpg similarity index 100% rename from examples/storytelling-chatbot/frontend/public/bg.jpg rename to examples/storytelling-chatbot/client/public/bg.jpg diff --git a/examples/storytelling-chatbot/frontend/tailwind.config.ts b/examples/storytelling-chatbot/client/tailwind.config.ts similarity index 100% rename from examples/storytelling-chatbot/frontend/tailwind.config.ts rename to examples/storytelling-chatbot/client/tailwind.config.ts diff --git a/examples/storytelling-chatbot/frontend/tsconfig.json b/examples/storytelling-chatbot/client/tsconfig.json similarity index 100% rename from examples/storytelling-chatbot/frontend/tsconfig.json rename to examples/storytelling-chatbot/client/tsconfig.json diff --git a/examples/storytelling-chatbot/server/.dockerignore b/examples/storytelling-chatbot/server/.dockerignore new file mode 100644 index 000000000..4c6d90950 --- /dev/null +++ b/examples/storytelling-chatbot/server/.dockerignore @@ -0,0 +1,2 @@ +client/node_modules +client/out \ No newline at end of file diff --git a/examples/storytelling-chatbot/Dockerfile b/examples/storytelling-chatbot/server/Dockerfile similarity index 87% rename from examples/storytelling-chatbot/Dockerfile rename to examples/storytelling-chatbot/server/Dockerfile index 5d762cd14..46da11124 100644 --- a/examples/storytelling-chatbot/Dockerfile +++ b/examples/storytelling-chatbot/server/Dockerfile @@ -44,11 +44,11 @@ COPY ./requirements.txt requirements.txt RUN pip3 install --no-cache-dir --upgrade -r requirements.txt # Copy everything else -COPY --chown=user ./src/ src/ +COPY --chown=user ./server/ server/ -# Copy frontend app and build -COPY --chown=user ./frontend/ frontend/ -RUN cd frontend && npm install && npm run build +# Copy client app and build +COPY --chown=user ./client/ client/ +RUN cd client && npm install && npm run build # Start the FastAPI server -CMD python3 src/bot_runner.py --port ${FAST_API_PORT} \ No newline at end of file +CMD python3 server/bot_runner.py --port ${FAST_API_PORT} \ No newline at end of file diff --git a/examples/storytelling-chatbot/src/assets/book1.png b/examples/storytelling-chatbot/server/assets/book1.png similarity index 100% rename from examples/storytelling-chatbot/src/assets/book1.png rename to examples/storytelling-chatbot/server/assets/book1.png diff --git a/examples/storytelling-chatbot/src/assets/book2.png b/examples/storytelling-chatbot/server/assets/book2.png similarity index 100% rename from examples/storytelling-chatbot/src/assets/book2.png rename to examples/storytelling-chatbot/server/assets/book2.png diff --git a/examples/storytelling-chatbot/src/assets/ding.wav b/examples/storytelling-chatbot/server/assets/ding.wav similarity index 100% rename from examples/storytelling-chatbot/src/assets/ding.wav rename to examples/storytelling-chatbot/server/assets/ding.wav diff --git a/examples/storytelling-chatbot/src/assets/listening.wav b/examples/storytelling-chatbot/server/assets/listening.wav similarity index 100% rename from examples/storytelling-chatbot/src/assets/listening.wav rename to examples/storytelling-chatbot/server/assets/listening.wav diff --git a/examples/storytelling-chatbot/src/assets/talking.wav b/examples/storytelling-chatbot/server/assets/talking.wav similarity index 100% rename from examples/storytelling-chatbot/src/assets/talking.wav rename to examples/storytelling-chatbot/server/assets/talking.wav diff --git a/examples/storytelling-chatbot/src/bot.py b/examples/storytelling-chatbot/server/bot.py similarity index 100% rename from examples/storytelling-chatbot/src/bot.py rename to examples/storytelling-chatbot/server/bot.py diff --git a/examples/storytelling-chatbot/src/bot_runner.py b/examples/storytelling-chatbot/server/bot_runner.py similarity index 98% rename from examples/storytelling-chatbot/src/bot_runner.py rename to examples/storytelling-chatbot/server/bot_runner.py index 591b2d598..f09b12fac 100644 --- a/examples/storytelling-chatbot/src/bot_runner.py +++ b/examples/storytelling-chatbot/server/bot_runner.py @@ -57,7 +57,7 @@ app.add_middleware( ) # Mount the static directory -STATIC_DIR = "frontend/out" +STATIC_DIR = "client/out" # ------------ Fast API Routes ------------ # @@ -175,7 +175,7 @@ async def virtualize_bot(room_url: str, token: str): image = data[0]["config"]["image"] # Machine configuration - cmd = f"python src/bot.py -u {room_url} -t {token}" + cmd = f"python server/bot.py -u {room_url} -t {token}" cmd = cmd.split() worker_props = { "config": { diff --git a/examples/storytelling-chatbot/env.example b/examples/storytelling-chatbot/server/env.example similarity index 100% rename from examples/storytelling-chatbot/env.example rename to examples/storytelling-chatbot/server/env.example diff --git a/examples/storytelling-chatbot/src/processors.py b/examples/storytelling-chatbot/server/processors.py similarity index 100% rename from examples/storytelling-chatbot/src/processors.py rename to examples/storytelling-chatbot/server/processors.py diff --git a/examples/storytelling-chatbot/src/prompts.py b/examples/storytelling-chatbot/server/prompts.py similarity index 100% rename from examples/storytelling-chatbot/src/prompts.py rename to examples/storytelling-chatbot/server/prompts.py diff --git a/examples/storytelling-chatbot/requirements.txt b/examples/storytelling-chatbot/server/requirements.txt similarity index 100% rename from examples/storytelling-chatbot/requirements.txt rename to examples/storytelling-chatbot/server/requirements.txt diff --git a/examples/storytelling-chatbot/src/utils/helpers.py b/examples/storytelling-chatbot/server/utils/helpers.py similarity index 100% rename from examples/storytelling-chatbot/src/utils/helpers.py rename to examples/storytelling-chatbot/server/utils/helpers.py diff --git a/pyproject.toml b/pyproject.toml index 6d1cf840c..ecddb0902 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ canonical = [ "aiofiles~=24.1.0" ] cartesia = [ "cartesia~=1.4.0", "websockets~=13.1" ] cerebras = [] deepseek = [] -daily = [ "daily-python~=0.18.0" ] +daily = [ "daily-python~=0.18.1" ] deepgram = [ "deepgram-sdk~=3.8.0" ] elevenlabs = [ "websockets~=13.1" ] fal = [ "fal-client~=0.5.9" ] @@ -78,7 +78,7 @@ perplexity = [] playht = [ "pyht~=0.1.12", "websockets~=13.1" ] qwen = [] rime = [ "websockets~=13.1" ] -riva = [ "nvidia-riva-client~=2.19.0" ] +riva = [ "nvidia-riva-client~=2.19.1" ] sentry = [ "sentry-sdk~=2.23.1" ] local-smart-turn = [ "coremltools>=8.0", "transformers", "torch==2.5.0", "torchaudio==2.5.0" ] remote-smart-turn = [] diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index e6d1d400a..57ade7484 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -68,6 +68,8 @@ class RimeTTSService(AudioContextWordTTSService): language: Optional[Language] = Language.EN speed_alpha: Optional[float] = 1.0 reduce_latency: Optional[bool] = False + pause_between_brackets: Optional[bool] = False + phonemize_between_brackets: Optional[bool] = False def __init__( self, @@ -117,6 +119,8 @@ class RimeTTSService(AudioContextWordTTSService): else "eng", "speedAlpha": params.speed_alpha, "reduceLatency": params.reduce_latency, + "pauseBetweenBrackets": json.dumps(params.pause_between_brackets), + "phonemizeBetweenBrackets": json.dumps(params.phonemize_between_brackets), } # State tracking diff --git a/src/pipecat/services/riva/stt.py b/src/pipecat/services/riva/stt.py index 6328bcb65..2b263f03e 100644 --- a/src/pipecat/services/riva/stt.py +++ b/src/pipecat/services/riva/stt.py @@ -5,7 +5,7 @@ # import asyncio -from typing import AsyncGenerator, Optional +from typing import AsyncGenerator, List, Mapping, Optional from loguru import logger from pydantic import BaseModel @@ -13,12 +13,13 @@ from pydantic import BaseModel from pipecat.frames.frames import ( CancelFrame, EndFrame, + ErrorFrame, Frame, InterimTranscriptionFrame, StartFrame, TranscriptionFrame, ) -from pipecat.services.stt_service import STTService +from pipecat.services.stt_service import SegmentedSTTService, STTService from pipecat.transcriptions.language import Language from pipecat.utils.time import time_now_iso8601 @@ -31,7 +32,59 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") -class ParakeetSTTService(STTService): +def language_to_riva_language(language: Language) -> Optional[str]: + """Maps Language enum to Riva ASR language codes. + + Source: + https://docs.nvidia.com/deeplearning/riva/user-guide/docs/asr/asr-riva-build-table.html?highlight=fr%20fr + + Args: + language: Language enum value. + + Returns: + Optional[str]: Riva language code or None if not supported. + """ + language_map = { + # Arabic + Language.AR: "ar-AR", + # English + Language.EN: "en-US", # Default to US + Language.EN_US: "en-US", + Language.EN_GB: "en-GB", + # French + Language.FR: "fr-FR", + Language.FR_FR: "fr-FR", + # German + Language.DE: "de-DE", + Language.DE_DE: "de-DE", + # Hindi + Language.HI: "hi-IN", + Language.HI_IN: "hi-IN", + # Italian + Language.IT: "it-IT", + Language.IT_IT: "it-IT", + # Japanese + Language.JA: "ja-JP", + Language.JA_JP: "ja-JP", + # Korean + Language.KO: "ko-KR", + Language.KO_KR: "ko-KR", + # Portuguese + Language.PT: "pt-BR", # Default to Brazilian + Language.PT_BR: "pt-BR", + # Russian + Language.RU: "ru-RU", + Language.RU_RU: "ru-RU", + # Spanish + Language.ES: "es-ES", # Default to Spain + Language.ES_ES: "es-ES", + Language.ES_US: "es-US", # US Spanish + } + + return language_map.get(language) + + +class RivaSTTService(STTService): class InputParams(BaseModel): language: Optional[Language] = Language.EN_US @@ -40,7 +93,10 @@ class ParakeetSTTService(STTService): *, api_key: str, server: str = "grpc.nvcf.nvidia.com:443", - function_id: str = "1598d209-5e27-4d3c-8079-4751568b1081", + model_function_map: Mapping[str, str] = { + "function_id": "1598d209-5e27-4d3c-8079-4751568b1081", + "model_name": "parakeet-ctc-1.1b-asr", + }, sample_rate: Optional[int] = None, params: InputParams = InputParams(), **kwargs, @@ -48,7 +104,7 @@ class ParakeetSTTService(STTService): super().__init__(sample_rate=sample_rate, **kwargs) self._api_key = api_key self._profanity_filter = False - self._automatic_punctuation = False + self._automatic_punctuation = True self._no_verbatim_transcripts = False self._language_code = params.language self._boosted_lm_words = None @@ -60,11 +116,12 @@ class ParakeetSTTService(STTService): self._stop_history_eou = -1 self._stop_threshold_eou = -1.0 self._custom_configuration = "" + self._function_id = model_function_map.get("function_id") - self.set_model_name("parakeet-ctc-1.1b-asr") + self.set_model_name(model_function_map.get("model_name")) metadata = [ - ["function-id", function_id], + ["function-id", self._function_id], ["authorization", f"Bearer {api_key}"], ] auth = riva.client.Auth(None, True, server, metadata) @@ -79,6 +136,13 @@ class ParakeetSTTService(STTService): def can_generate_metrics(self) -> bool: return False + async def set_model(self, model: str): + logger.warning(f"Cannot set model after initialization. Set model and function id like so:") + example = {"function_id": "", "model_name": ""} + logger.warning( + f"{self.__class__.__name__}(api_key=, model_function_map={example})" + ) + async def start(self, frame: StartFrame): await super().start(frame) @@ -196,3 +260,262 @@ class ParakeetSTTService(STTService): def __iter__(self): return self + + +class RivaSegmentedSTTService(SegmentedSTTService): + """Speech-to-text service using NVIDIA Riva's offline/batch models. + + By default, his service uses NVIDIA's Riva Canary ASR API to perform speech-to-text + transcription on audio segments. It inherits from SegmentedSTTService to handle + audio buffering and speech detection. + + Args: + api_key: NVIDIA API key for authentication + server: Riva server address (defaults to NVIDIA Cloud Function endpoint) + model_function_map: Mapping of model name and its corresponding NVIDIA Cloud Function ID + sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate + params: Additional configuration parameters for Riva + **kwargs: Additional arguments passed to SegmentedSTTService + """ + + class InputParams(BaseModel): + language: Optional[Language] = Language.EN_US + profanity_filter: bool = False + automatic_punctuation: bool = True + verbatim_transcripts: bool = False + boosted_lm_words: Optional[List[str]] = None + boosted_lm_score: float = 4.0 + + def __init__( + self, + *, + api_key: str, + server: str = "grpc.nvcf.nvidia.com:443", + model_function_map: Mapping[str, str] = { + "function_id": "ee8dc628-76de-4acc-8595-1836e7e857bd", + "model_name": "canary-1b-asr", + }, + sample_rate: Optional[int] = None, + params: InputParams = InputParams(), + **kwargs, + ): + super().__init__(sample_rate=sample_rate, **kwargs) + + # Set model name + self.set_model_name(model_function_map.get("model_name")) + + # Initialize Riva settings + self._api_key = api_key + self._server = server + self._function_id = model_function_map.get("function_id") + self._model_name = model_function_map.get("model_name") + + # Store the language as a Language enum and as a string + self._language_enum = params.language or Language.EN_US + self._language = self.language_to_service_language(self._language_enum) or "en-US" + + # Configure transcription parameters + self._profanity_filter = params.profanity_filter + self._automatic_punctuation = params.automatic_punctuation + self._verbatim_transcripts = params.verbatim_transcripts + self._boosted_lm_words = params.boosted_lm_words + self._boosted_lm_score = params.boosted_lm_score + + # Voice activity detection thresholds (use Riva defaults) + self._start_history = -1 + self._start_threshold = -1.0 + self._stop_history = -1 + self._stop_threshold = -1.0 + self._stop_history_eou = -1 + self._stop_threshold_eou = -1.0 + self._custom_configuration = "" + + # Create Riva client + self._config = None + self._asr_service = None + self._settings = {"language": self._language_enum} + + def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert pipecat Language enum to Riva's language code.""" + return language_to_riva_language(language) + + def _initialize_client(self): + """Initialize the Riva ASR client with authentication metadata.""" + if self._asr_service is not None: + return + + # Set up authentication metadata for NVIDIA Cloud Functions + metadata = [ + ["function-id", self._function_id], + ["authorization", f"Bearer {self._api_key}"], + ] + + # Create authenticated client + auth = riva.client.Auth(None, True, self._server, metadata) + self._asr_service = riva.client.ASRService(auth) + + logger.info(f"Initialized RivaSegmentedSTTService with model: {self.model_name}") + + def _create_recognition_config(self): + """Create the Riva ASR recognition configuration.""" + # Create base configuration + config = riva.client.RecognitionConfig( + language_code=self._language, # Now using the string, not a tuple + max_alternatives=1, + profanity_filter=self._profanity_filter, + enable_automatic_punctuation=self._automatic_punctuation, + verbatim_transcripts=self._verbatim_transcripts, + ) + + # Add word boosting if specified + if self._boosted_lm_words: + riva.client.add_word_boosting_to_config( + config, self._boosted_lm_words, self._boosted_lm_score + ) + + # Add voice activity detection parameters + riva.client.add_endpoint_parameters_to_config( + config, + self._start_history, + self._start_threshold, + self._stop_history, + self._stop_history_eou, + self._stop_threshold, + self._stop_threshold_eou, + ) + + # Add any custom configuration + if self._custom_configuration: + riva.client.add_custom_configuration_to_config(config, self._custom_configuration) + + return config + + def can_generate_metrics(self) -> bool: + """Indicates whether this service can generate processing metrics.""" + return True + + async def set_model(self, model: str): + logger.warning(f"Cannot set model after initialization. Set model and function id like so:") + example = {"function_id": "", "model_name": ""} + logger.warning( + f"{self.__class__.__name__}(api_key=, model_function_map={example})" + ) + + async def start(self, frame: StartFrame): + """Initialize the service when the pipeline starts.""" + await super().start(frame) + self._initialize_client() + self._config = self._create_recognition_config() + + async def set_language(self, language: Language): + """Set the language for the STT service.""" + logger.info(f"Switching STT language to: [{language}]") + self._language_enum = language + self._language = self.language_to_service_language(language) or "en-US" + self._settings["language"] = language + + # Update configuration with new language + if self._config: + self._config.language_code = self._language + + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Transcribe an audio segment. + + Args: + audio: Raw audio bytes in WAV format (already converted by base class). + + Yields: + Frame: TranscriptionFrame containing the transcribed text. + """ + try: + await self.start_processing_metrics() + await self.start_ttfb_metrics() + + # Make sure the client is initialized + if self._asr_service is None: + self._initialize_client() + + # Make sure the config is created + if self._config is None: + self._config = self._create_recognition_config() + + # Type assertion to satisfy the IDE + assert self._asr_service is not None, "ASR service not initialized" + assert self._config is not None, "Recognition config not created" + + # Process audio with Riva ASR - explicitly request non-future response + raw_response = self._asr_service.offline_recognize(audio, self._config, future=False) + + await self.stop_ttfb_metrics() + await self.stop_processing_metrics() + + # Process the response - handle different possible return types + try: + # If it's a future-like object, get the result + if hasattr(raw_response, "result"): + response = raw_response.result() + else: + response = raw_response + + # Process transcription results + transcription_found = False + + # Now we can safely check results + # Type hint for the IDE + results = getattr(response, "results", []) + + for result in results: + alternatives = getattr(result, "alternatives", []) + if alternatives: + text = alternatives[0].transcript.strip() + if text: + logger.debug(f"Transcription: [{text}]") + yield TranscriptionFrame( + text, "", time_now_iso8601(), self._language_enum + ) + transcription_found = True + + if not transcription_found: + logger.debug("No transcription results found in Riva response") + + except AttributeError as ae: + logger.error(f"Unexpected response structure from Riva: {ae}") + yield ErrorFrame(f"Unexpected Riva response format: {str(ae)}") + + except Exception as e: + logger.exception(f"Riva Canary ASR error: {e}") + yield ErrorFrame(f"Riva Canary ASR error: {str(e)}") + + +class ParakeetSTTService(RivaSTTService): + """Deprecated: Use RivaSTTService instead.""" + + def __init__( + self, + *, + api_key: str, + server: str = "grpc.nvcf.nvidia.com:443", + model_function_map: Mapping[str, str] = { + "function_id": "1598d209-5e27-4d3c-8079-4751568b1081", + "model_name": "parakeet-ctc-1.1b-asr", + }, + sample_rate: Optional[int] = None, + params: RivaSTTService.InputParams = RivaSTTService.InputParams(), # Use parent class's type + **kwargs, + ): + super().__init__( + api_key=api_key, + server=server, + model_function_map=model_function_map, + sample_rate=sample_rate, + params=params, + **kwargs, + ) + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "`ParakeetSTTService` is deprecated, use `RivaSTTService` instead.", + DeprecationWarning, + ) diff --git a/src/pipecat/services/riva/tts.py b/src/pipecat/services/riva/tts.py index 0fd0c3ce0..4fc8b8489 100644 --- a/src/pipecat/services/riva/tts.py +++ b/src/pipecat/services/riva/tts.py @@ -5,7 +5,11 @@ # import asyncio -from typing import AsyncGenerator, Optional +import os +from typing import AsyncGenerator, Mapping, Optional + +# Suppress gRPC fork warnings +os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false" from loguru import logger from pydantic import BaseModel @@ -27,10 +31,10 @@ except ModuleNotFoundError as e: logger.error("In order to use NVIDIA Riva TTS, you need to `pip install pipecat-ai[riva]`.") raise Exception(f"Missing module: {e}") -FASTPITCH_TIMEOUT_SECS = 5 +RIVA_TTS_TIMEOUT_SECS = 5 -class FastPitchTTSService(TTSService): +class RivaTTSService(TTSService): class InputParams(BaseModel): language: Optional[Language] = Language.EN_US quality: Optional[int] = 20 @@ -38,11 +42,14 @@ class FastPitchTTSService(TTSService): def __init__( self, *, - api_key: str, + api_key: str = None, server: str = "grpc.nvcf.nvidia.com:443", - voice_id: str = "English-US.Female-1", + voice_id: str = "Magpie-Multilingual.EN-US.Ray", sample_rate: Optional[int] = None, - function_id: str = "0149dedb-2be8-4195-b9a0-e57e0e14f972", + model_function_map: Mapping[str, str] = { + "function_id": "877104f7-e885-42b9-8de8-f6e4c6303969", + "model_name": "magpie-tts-multilingual", + }, params: InputParams = InputParams(), **kwargs, ): @@ -51,12 +58,13 @@ class FastPitchTTSService(TTSService): self._voice_id = voice_id self._language_code = params.language self._quality = params.quality + self._function_id = model_function_map.get("function_id") - self.set_model_name("fastpitch-hifigan-tts") + self.set_model_name(model_function_map.get("model_name")) self.set_voice(voice_id) metadata = [ - ["function-id", function_id], + ["function-id", self._function_id], ["authorization", f"Bearer {api_key}"], ] auth = riva.client.Auth(None, True, server, metadata) @@ -68,6 +76,13 @@ class FastPitchTTSService(TTSService): riva.client.proto.riva_tts_pb2.RivaSynthesisConfigRequest() ) + async def set_model(self, model: str): + logger.warning(f"Cannot set model after initialization. Set model and function id like so:") + example = {"function_id": "", "model_name": ""} + logger.warning( + f"{self.__class__.__name__}(api_key=, model_function_map={example})" + ) + async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: def read_audio_responses(queue: asyncio.Queue): def add_response(r): @@ -100,7 +115,7 @@ class FastPitchTTSService(TTSService): await asyncio.to_thread(read_audio_responses, queue) # Wait for the thread to start. - resp = await asyncio.wait_for(queue.get(), FASTPITCH_TIMEOUT_SECS) + resp = await asyncio.wait_for(queue.get(), RIVA_TTS_TIMEOUT_SECS) while resp: await self.stop_ttfb_metrics() frame = TTSAudioRawFrame( @@ -109,9 +124,46 @@ class FastPitchTTSService(TTSService): num_channels=1, ) yield frame - resp = await asyncio.wait_for(queue.get(), FASTPITCH_TIMEOUT_SECS) + resp = await asyncio.wait_for(queue.get(), RIVA_TTS_TIMEOUT_SECS) except asyncio.TimeoutError: logger.error(f"{self} timeout waiting for audio response") await self.start_tts_usage_metrics(text) yield TTSStoppedFrame() + + +class FastPitchTTSService(RivaTTSService): + class InputParams(BaseModel): + language: Optional[Language] = Language.EN_US + quality: Optional[int] = 20 + + def __init__( + self, + *, + api_key: str = None, + server: str = "grpc.nvcf.nvidia.com:443", + voice_id: str = "English-US.Female-1", + sample_rate: Optional[int] = None, + model_function_map: Mapping[str, str] = { + "function_id": "0149dedb-2be8-4195-b9a0-e57e0e14f972", + "model_name": "fastpitch-hifigan-tts", + }, + params: InputParams = InputParams(), + **kwargs, + ): + super().__init__( + api_key=api_key, + voice_id=voice_id, + sample_rate=sample_rate, + model_function_map=model_function_map, + params=params, + **kwargs, + ) + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "`FastPitchTTSService` is deprecated, use `RivaTTSService` instead.", + DeprecationWarning, + ) diff --git a/src/pipecat/services/simli/video.py b/src/pipecat/services/simli/video.py index 0fb490914..121801e4b 100644 --- a/src/pipecat/services/simli/video.py +++ b/src/pipecat/services/simli/video.py @@ -64,13 +64,16 @@ class SimliVideoService(FrameProcessor): async for audio_frame in self._simli_client.getAudioStreamIterator(): resampled_frames = self._pipecat_resampler.resample(audio_frame) for resampled_frame in resampled_frames: - await self.push_frame( - TTSAudioRawFrame( - audio=resampled_frame.to_ndarray().tobytes(), - sample_rate=self._pipecat_resampler.rate, - num_channels=1, - ), - ) + audio_array = resampled_frame.to_ndarray() + # Only push frame is there is audio (e.g. not silence) + if audio_array.any(): + await self.push_frame( + TTSAudioRawFrame( + audio=audio_array.tobytes(), + sample_rate=self._pipecat_resampler.rate, + num_channels=1, + ), + ) async def _consume_and_process_video(self): await self._pipecat_resampler_event.wait() diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py index a3be44073..a5264f622 100644 --- a/src/pipecat/transports/base_output.py +++ b/src/pipecat/transports/base_output.py @@ -369,7 +369,7 @@ class BaseOutputTransport(FrameProcessor): # def _create_audio_task(self): - if not self._audio_task and self._params.audio_out_enabled: + if not self._audio_task: self._audio_queue = asyncio.Queue() self._audio_task = self._transport.create_task(self._audio_task_handler()) @@ -380,7 +380,9 @@ class BaseOutputTransport(FrameProcessor): async def _bot_started_speaking(self): if not self._bot_speaking: - logger.debug(f"Bot [{self._destination}] started speaking") + logger.debug( + f"Bot{f' [{self._destination}]' if self._destination else ''} started speaking" + ) downstream_frame = BotStartedSpeakingFrame() downstream_frame.transport_destination = self._destination @@ -393,7 +395,9 @@ class BaseOutputTransport(FrameProcessor): async def _bot_stopped_speaking(self): if self._bot_speaking: - logger.debug(f"Bot [{self._destination}] stopped speaking") + logger.debug( + f"Bot{f' [{self._destination}]' if self._destination else ''} stopped speaking" + ) downstream_frame = BotStoppedSpeakingFrame() downstream_frame.transport_destination = self._destination diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index 08a2a6030..1684d572a 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -11,14 +11,6 @@ from dataclasses import dataclass from typing import Any, Awaitable, Callable, Dict, Mapping, Optional import aiohttp -from daily import ( - AudioData, - CustomAudioSource, - VideoFrame, - VirtualCameraDevice, - VirtualMicrophoneDevice, - VirtualSpeakerDevice, -) from loguru import logger from pydantic import BaseModel @@ -50,7 +42,17 @@ from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.utils.asyncio import BaseTaskManager try: - from daily import CallClient, Daily, EventHandler + from daily import ( + AudioData, + CallClient, + CustomAudioSource, + Daily, + EventHandler, + VideoFrame, + VirtualCameraDevice, + VirtualMicrophoneDevice, + VirtualSpeakerDevice, + ) except ModuleNotFoundError as e: logger.error(f"Exception: {e}") logger.error(