diff --git a/dev-requirements.txt b/dev-requirements.txt index c706d8fe6..92b6ec4d3 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,5 @@ build~=1.2.1 -grpcio-tools~=1.62.2 +grpcio-tools~=1.65.4 pip-tools~=7.4.1 pyright~=1.1.376 pytest~=8.3.2 diff --git a/examples/foundational/01c-fastpitch.py b/examples/foundational/01c-fastpitch.py new file mode 100644 index 000000000..49499cdfc --- /dev/null +++ b/examples/foundational/01c-fastpitch.py @@ -0,0 +1,56 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import aiohttp +import os +import sys + +from pipecat.frames.frames import EndFrame, TTSSpeakFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.task import PipelineTask +from pipecat.pipeline.runner import PipelineRunner +from pipecat.services.riva import FastpitchTTSService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +from runner import configure + +from loguru import logger + +from dotenv import load_dotenv + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, _) = await configure(session) + + transport = DailyTransport( + room_url, None, "Say One Thing", DailyParams(audio_out_enabled=True) + ) + + tts = FastpitchTTSService(api_key=os.getenv("NVIDIA_API_KEY")) + + runner = PipelineRunner() + + task = PipelineTask(Pipeline([tts, transport.output()])) + + # Register an event handler so we can play the audio when the + # participant joins. + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + participant_name = participant.get("info", {}).get("userName", "") + await task.queue_frames([TTSSpeakFrame(f"Aloha, {participant_name}!"), EndFrame()]) + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/foundational/07r-interruptible-riva-nim.py b/examples/foundational/07r-interruptible-riva-nim.py new file mode 100644 index 000000000..09d261bc3 --- /dev/null +++ b/examples/foundational/07r-interruptible-riva-nim.py @@ -0,0 +1,92 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMMessagesFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.nim import NimLLMService +from pipecat.services.riva import FastpitchTTSService, ParakeetSTTService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, _) = await configure(session) + + transport = DailyTransport( + room_url, + None, + "Respond bot", + DailyParams( + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) + + stt = ParakeetSTTService(api_key=os.getenv("NVIDIA_API_KEY")) + + llm = NimLLMService( + api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.1-405b-instruct" + ) + + tts = FastpitchTTSService(api_key=os.getenv("NVIDIA_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True)) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMMessagesFrame(messages)]) + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/pyproject.toml b/pyproject.toml index b2bd0c48f..549ecece8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ "Markdown~=3.7", "numpy~=1.26.4", "Pillow~=10.4.0", - "protobuf~=4.25.4", + "protobuf~=5.26.1", "pydantic~=2.8.2", "pyloudnorm~=0.1.1", "resampy~=0.4.3", @@ -48,7 +48,7 @@ elevenlabs = [ "websockets~=13.1" ] examples = [ "python-dotenv~=1.0.1", "flask~=3.0.3", "flask_cors~=4.0.1" ] fal = [ "fal-client~=0.4.1" ] gladia = [ "websockets~=13.1" ] -google = [ "google-generativeai~=0.8.3", "google-cloud-texttospeech~=2.17.2" ] +google = [ "google-generativeai~=0.8.3", "google-cloud-texttospeech~=2.21.1" ] grok = [ "openai~=1.50.2" ] groq = [ "openai~=1.50.2" ] gstreamer = [ "pygobject~=3.48.2" ] @@ -63,7 +63,8 @@ nim = [ "openai~=1.50.2" ] noisereduce = [ "noisereduce~=3.0.3" ] openai = [ "openai~=1.50.2", "websockets~=13.1", "python-deepcompare~=1.0.1" ] openpipe = [ "openpipe~=4.38.0" ] -playht = [ "pyht~=0.1.4", "websockets~=13.1" ] +playht = [ "pyht~=0.1.8", "websockets~=13.1" ] +riva = [ "nvidia-riva-client~=2.17.0" ] silero = [ "onnxruntime~=1.19.2" ] soundfile = [ "soundfile~=0.12.1" ] together = [ "openai~=1.50.2" ] diff --git a/src/pipecat/services/riva.py b/src/pipecat/services/riva.py new file mode 100644 index 000000000..0bbaee0ef --- /dev/null +++ b/src/pipecat/services/riva.py @@ -0,0 +1,266 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +from typing import AsyncGenerator, Optional + +from loguru import logger +from pydantic.main import BaseModel + +from pipecat.frames.frames import ( + CancelFrame, + EndFrame, + Frame, + InterimTranscriptionFrame, + StartFrame, + TranscriptionFrame, + TTSAudioRawFrame, + TTSStartedFrame, + TTSStoppedFrame, +) +from pipecat.services.ai_services import STTService, TTSService +from pipecat.transcriptions.language import Language +from pipecat.utils.time import time_now_iso8601 + +try: + import riva.client + +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error( + "In order to use nvidia riva TTS or STT, you need to `pip install pipecat-ai[riva]`. Also, set `NVIDIA_API_KEY` environment variable." + ) + raise Exception(f"Missing module: {e}") + + +class FastpitchTTSService(TTSService): + class InputParams(BaseModel): + language: Optional[Language] = Language.EN_US + quality: Optional[int] = 20 + + def __init__( + self, + *, + api_key: str, + server: str = "grpc.nvcf.nvidia.com:443", + voice_id: str = "English-US.Female-1", + sample_rate: int = 24000, + function_id: str = "0149dedb-2be8-4195-b9a0-e57e0e14f972", + params: InputParams = InputParams(), + **kwargs, + ): + super().__init__(sample_rate=sample_rate, **kwargs) + self._api_key = api_key + self._voice_id = voice_id + self._sample_rate = sample_rate + self._language_code = params.language + self._quality = params.quality + + self.set_model_name("fastpitch-hifigan-tts") + self.set_voice(voice_id) + + metadata = [ + ["function-id", function_id], + ["authorization", f"Bearer {api_key}"], + ] + auth = riva.client.Auth(None, True, server, metadata) + + self._service = riva.client.SpeechSynthesisService(auth) + + # warm up the service + config_response = self._service.stub.GetRivaSynthesisConfig( + riva.client.proto.riva_tts_pb2.RivaSynthesisConfigRequest() + ) + + async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + def read_audio_responses(): + try: + responses = self._service.synthesize_online( + text, + self._voice_id, + self._language_code, + sample_rate_hz=self._sample_rate, + audio_prompt_file=None, + quality=self._quality, + custom_dictionary={}, + ) + return responses + except Exception as e: + logger.error(f"{self} exception: {e}") + return [] + + await self.start_ttfb_metrics() + yield TTSStartedFrame() + + logger.debug(f"Generating TTS: [{text}]") + responses = await asyncio.to_thread(read_audio_responses) + + for resp in responses: + await self.stop_ttfb_metrics() + + frame = TTSAudioRawFrame( + audio=resp.audio, + sample_rate=self._sample_rate, + num_channels=1, + ) + yield frame + + await self.start_tts_usage_metrics(text) + yield TTSStoppedFrame() + + +class ParakeetSTTService(STTService): + class InputParams(BaseModel): + language: Optional[Language] = Language.EN_US + + def __init__( + self, + *, + api_key: str, + server: str = "grpc.nvcf.nvidia.com:443", + function_id: str = "1598d209-5e27-4d3c-8079-4751568b1081", + params: InputParams = InputParams(), + **kwargs, + ): + super().__init__(**kwargs) + self._api_key = api_key + self._profanity_filter = False + self._automatic_punctuation = False + self._no_verbatim_transcripts = False + self._language_code = params.language + self._boosted_lm_words = None + self._boosted_lm_score = 4.0 + self._start_history = -1 + self._start_threshold = -1.0 + self._stop_history = -1 + self._stop_threshold = -1.0 + self._stop_history_eou = -1 + self._stop_threshold_eou = -1.0 + self._custom_configuration = "" + self._sample_rate: int = 16000 + + self.set_model_name("parakeet-ctc-1.1b-asr") + + metadata = [ + ["function-id", function_id], + ["authorization", f"Bearer {api_key}"], + ] + auth = riva.client.Auth(None, True, server, metadata) + + self._asr_service = riva.client.ASRService(auth) + + config = riva.client.StreamingRecognitionConfig( + config=riva.client.RecognitionConfig( + encoding=riva.client.AudioEncoding.LINEAR_PCM, + language_code=self._language_code, + model="", + max_alternatives=1, + profanity_filter=self._profanity_filter, + enable_automatic_punctuation=self._automatic_punctuation, + verbatim_transcripts=not self._no_verbatim_transcripts, + sample_rate_hertz=self._sample_rate, + audio_channel_count=1, + ), + interim_results=True, + ) + riva.client.add_word_boosting_to_config( + config, self._boosted_lm_words, self._boosted_lm_score + ) + riva.client.add_endpoint_parameters_to_config( + config, + self._start_history, + self._start_threshold, + self._stop_history, + self._stop_history_eou, + self._stop_threshold, + self._stop_threshold_eou, + ) + riva.client.add_custom_configuration_to_config(config, self._custom_configuration) + self._config = config + + self._queue = asyncio.Queue() + + def can_generate_metrics(self) -> bool: + return False + + async def start(self, frame: StartFrame): + await super().start(frame) + self._thread_task = self.get_event_loop().create_task(self._thread_task_handler()) + self._response_task = self.get_event_loop().create_task(self._response_task_handler()) + self._response_queue = asyncio.Queue() + + async def stop(self, frame: EndFrame): + await super().stop(frame) + await self._stop_tasks() + + async def cancel(self, frame: CancelFrame): + await super().cancel(frame) + await self._stop_tasks() + + async def _stop_tasks(self): + self._thread_task.cancel() + await self._thread_task + self._response_task.cancel() + await self._response_task + + def _response_handler(self): + responses = self._asr_service.streaming_response_generator( + audio_chunks=self, + streaming_config=self._config, + ) + for response in responses: + if not response.results: + continue + asyncio.run_coroutine_threadsafe( + self._response_queue.put(response), self.get_event_loop() + ) + + async def _thread_task_handler(self): + try: + self._thread_running = True + await asyncio.to_thread(self._response_handler) + except asyncio.CancelledError: + self._thread_running = False + pass + + async def _handle_response(self, response): + for result in response.results: + if result and not result.alternatives: + continue + + transcript = result.alternatives[0].transcript + if transcript and len(transcript) > 0: + await self.stop_ttfb_metrics() + if result.is_final: + await self.stop_processing_metrics() + await self.push_frame( + TranscriptionFrame(transcript, "", time_now_iso8601(), None) + ) + else: + await self.push_frame( + InterimTranscriptionFrame(transcript, "", time_now_iso8601(), None) + ) + + async def _response_task_handler(self): + while True: + try: + response = await self._response_queue.get() + await self._handle_response(response) + except asyncio.CancelledError: + break + + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + await self._queue.put(audio) + yield None + + def __next__(self) -> bytes: + if not self._thread_running: + raise StopIteration + future = asyncio.run_coroutine_threadsafe(self._queue.get(), self.get_event_loop()) + return future.result() + + def __iter__(self): + return self diff --git a/test-requirements.txt b/test-requirements.txt index 62be00385..f974121b0 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -7,8 +7,8 @@ deepgram-sdk~=3.5.0 fal-client~=0.4.1 fastapi~=0.115.0 faster-whisper~=1.0.3 -google-cloud-texttospeech~=2.17.2 -google-generativeai~=0.7.2 +google-cloud-texttospeech~=2.21.1 +google-generativeai~=0.8.3 langchain~=0.2.14 livekit~=0.13.1 lmnt~=1.1.4