diff --git a/CHANGELOG.md b/CHANGELOG.md index c3d6b32b5..60f30806f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added `XTTSService`. This is a local Text-To-Speech service. + See https://github.com/coqui-ai/TTS + - It is now possible to specify a Silero VAD version when using `SileroVADAnalyzer` or `SileroVAD`. diff --git a/examples/foundational/07i-interruptible-xtts.py b/examples/foundational/07i-interruptible-xtts.py new file mode 100644 index 000000000..73414f294 --- /dev/null +++ b/examples/foundational/07i-interruptible-xtts.py @@ -0,0 +1,96 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import aiohttp +import os +import sys + +from pipecat.frames.frames import LLMMessagesFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_response import ( + LLMAssistantResponseAggregator, LLMUserResponseAggregator) +from pipecat.services.deepgram import DeepgramSTTService, DeepgramTTSService +from pipecat.services.openai import OpenAILLMService +from pipecat.services.xtts import XTTSService +from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.vad.silero import SileroVADAnalyzer + +from runner import configure + +from loguru import logger + +from dotenv import load_dotenv +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def main(room_url: str, token): + async with aiohttp.ClientSession() as session: + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ) + ) + + tts = XTTSService( + aiohttp_session=session, + voice_id="Claribel Dervla", + language="en", + base_url="http://localhost:8000" + ) + + llm = OpenAILLMService( + api_key=os.getenv("OPENAI_API_KEY"), + model="gpt-4o") + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + tma_in = LLMUserResponseAggregator(messages) + tma_out = LLMAssistantResponseAggregator(messages) + + pipeline = Pipeline([ + transport.input(), # Transport user input + tma_in, # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + tma_out # Assistant spoken responses + ]) + + task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True)) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + messages.append( + {"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMMessagesFrame(messages)]) + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + (url, token) = configure() + asyncio.run(main(url, token)) diff --git a/src/pipecat/services/xtts.py b/src/pipecat/services/xtts.py index faf93f7cb..67aa86ccc 100644 --- a/src/pipecat/services/xtts.py +++ b/src/pipecat/services/xtts.py @@ -24,13 +24,14 @@ except ModuleNotFoundError as e: logger.error("In order to use XTTS, you need to `pip install pipecat-ai[xtts]`.") raise Exception(f"Missing module: {e}") -##### -## The server below can connect to XTTS through a local running docker -## -## Docker command: $ docker run --gpus=all -e COQUI_TOS_AGREED=1 --rm -p 8000:80 ghcr.io/coqui-ai/xtts-streaming-server:latest-cuda121 -## -## You can find more information on the official repo: https://github.com/coqui-ai/xtts-streaming-server -#### + +# The server below can connect to XTTS through a local running docker +# +# Docker command: $ docker run --gpus=all -e COQUI_TOS_AGREED=1 --rm -p 8000:80 ghcr.io/coqui-ai/xtts-streaming-server:latest-cuda121 +# +# You can find more information on the official repo: +# https://github.com/coqui-ai/xtts-streaming-server + class XTTSService(TTSService): @@ -40,7 +41,7 @@ class XTTSService(TTSService): aiohttp_session: aiohttp.ClientSession, voice_id: str, language: str, - base_url:str, + base_url: str, **kwargs): super().__init__(**kwargs) @@ -58,9 +59,9 @@ class XTTSService(TTSService): embeddings = self._studio_speakers[self._voice_id] url = self._base_url + "/tts_stream" - - payload={ - "text": text.replace('.','').replace('*',''), + + payload = { + "text": text.replace('.', '').replace('*', ''), "language": self._language, "speaker_embedding": embeddings["speaker_embedding"], "gpt_cond_latent": embeddings["gpt_cond_latent"], @@ -76,7 +77,7 @@ class XTTSService(TTSService): logger.error(f"{self} error getting audio (status: {r.status}, error: {text})") yield ErrorFrame(f"Error getting audio (status: {r.status}, error: {text})") return - + buffer = bytearray() async for chunk in r.content.iter_chunked(1024): @@ -84,14 +85,14 @@ class XTTSService(TTSService): await self.stop_ttfb_metrics() # Append new chunk to the buffer buffer.extend(chunk) - + # Check if buffer has enough data for processing while len(buffer) >= 48000: # Assuming at least 0.5 seconds of audio data at 24000 Hz # Process the buffer up to a safe size for resampling process_data = buffer[:48000] # Remove processed data from buffer buffer = buffer[48000:] - + # Convert the byte data to numpy array for resampling audio_np = np.frombuffer(process_data, dtype=np.int16) # Resample the audio from 24000 Hz to 16000 Hz @@ -108,4 +109,4 @@ class XTTSService(TTSService): resampled_audio = resampy.resample(audio_np, 24000, 16000) resampled_audio_bytes = resampled_audio.astype(np.int16).tobytes() frame = AudioRawFrame(resampled_audio_bytes, 16000, 1) - yield frame \ No newline at end of file + yield frame