wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #658 from pipecat-ai/aleix/default-to-24000-sample-rate

Browse Source 
update TTS and transport output sample rate to 24000
This commit is contained in:
Aleix Conchillo Flaqué
2024-10-24 16:48:41 -07:00
committed by GitHub
parent d24c6185d8 60c8c2f6e9
commit 2ba753272a
19 changed files with 75 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGELOG.md
View File

@@ -13,6 +13,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
grained control of what media subscriptions you want for each participant in a
room.
### Changed
- Changed default output sample rate to 24000. This changes all TTS service to
output to 24000 and also the default output transport sample rate. This
improves audio quality at the cost of some extra bandwidth.
## [0.0.47] - 2024-10-22
### Added

2
examples/foundational/01b-livekit-audio.py
View File

@@ -81,7 +81,7 @@ async def main():
url=url,
token=token,
room_name=room_name,
params=LiveKitParams(audio_out_enabled=True, audio_out_sample_rate=16000),
params=LiveKitParams(audio_out_enabled=True),
)
tts = CartesiaTTSService(

1
examples/foundational/07e-interruptible-playht.py
View File

@@ -40,7 +40,6 @@ async def main():
"Respond bot",
DailyParams(
audio_out_enabled=True,
audio_out_sample_rate=16000,
transcription_enabled=True,
vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(),

1
examples/foundational/07f-interruptible-azure.py
View File

@@ -41,7 +41,6 @@ async def main():
"Respond bot",
DailyParams(
audio_out_enabled=True,
audio_out_sample_rate=16000,
vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,

1
examples/foundational/09-mirror.py
View File

@@ -63,6 +63,7 @@ async def main():
"Test",
DailyParams(
audio_in_enabled=True,
audio_in_sample_rate=24000,
audio_out_enabled=True,
camera_out_enabled=True,
camera_out_is_live=True,

2
examples/foundational/09a-local-mirror.py
View File

@@ -65,7 +65,7 @@ async def main():
tk_root.title("Local Mirror")
daily_transport = DailyTransport(
room_url, token, "Test", DailyParams(audio_in_enabled=True)
room_url, token, "Test", DailyParams(audio_in_enabled=True, audio_in_sample_rate=24000)
)
tk_transport = TkLocalTransport(

3
examples/foundational/12c-describe-video-anthropic.py
View File

@@ -78,9 +78,6 @@ async def main():
tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady
params=CartesiaTTSService.InputParams(
sample_rate=16000,
),
)
@transport.event_handler("on_first_participant_joined")

8
examples/foundational/15a-switch-languages.py
View File

@@ -10,7 +10,7 @@ import os
import sys
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import LLMMessagesFrame
from pipecat.frames.frames import LLMMessagesFrame, TTSUpdateSettingsFrame
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.parallel_pipeline import ParallelPipeline
from pipecat.pipeline.runner import PipelineRunner
@@ -19,7 +19,6 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.filters.function_filter import FunctionFilter
from pipecat.services.cartesia import CartesiaTTSService
from pipecat.services.openai import OpenAILLMService
from pipecat.services.whisper import Model, WhisperSTTService
from pipecat.transports.services.daily import DailyParams, DailyTransport
from openai.types.chat import ChatCompletionToolParam
@@ -61,16 +60,14 @@ async def main():
token,
"Pipecat",
DailyParams(
audio_in_enabled=True,
audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
),
)
stt = WhisperSTTService(model=Model.LARGE)
english_tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady
@@ -116,7 +113,6 @@ async def main():
pipeline = Pipeline(
[
transport.input(), # Transport user input
stt, # STT
context_aggregator.user(), # User responses
llm, # LLM
ParallelPipeline( # TTS (bot will speak the chosen language)

2
src/pipecat/audio/utils.py
View File

@@ -11,6 +11,8 @@ from scipy import signal
def resample_audio(audio: bytes, original_rate: int, target_rate: int) -> bytes:
if original_rate == target_rate:
return audio
audio_data = np.frombuffer(audio, dtype=np.int16)
num_samples = int(len(audio) * target_rate / original_rate)
resampled_audio = signal.resample(audio_data, num_samples)

2
src/pipecat/audio/vad/silero.py
View File

@@ -52,7 +52,7 @@ class SileroOnnxModel:
if sr not in self.sample_rates:
raise ValueError(
f"Supported sampling rates: {self.sample_rates} (or multiply of 16000)"
f"Supported sampling rates: {self.sample_rates} (or multiple of 16000)"
)
if sr / np.shape(x)[1] > 31.25:
raise ValueError("Input audio chunk is too short")

4
src/pipecat/services/ai_services.py
View File

@@ -205,7 +205,7 @@ class TTSService(AIService):
# if push_stop_frames is True, wait for this idle period before pushing TTSStoppedFrame
stop_frame_timeout_s: float = 1.0,
# TTS output sample rate
sample_rate: int = 16000,
sample_rate: int = 24000,
text_filter: Optional[BaseTextFilter] = None,
**kwargs,
):
@@ -514,7 +514,7 @@ class SegmentedSTTService(STTService):
min_volume: float = 0.6,
max_silence_secs: float = 0.3,
max_buffer_secs: float = 1.5,
sample_rate: int = 16000,
sample_rate: int = 24000,
num_channels: int = 1,
**kwargs,
):

42
src/pipecat/services/azure.py
View File

@@ -25,8 +25,14 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
URLImageRawFrame,
)
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.ai_services import ImageGenService, STTService, TTSService
from pipecat.services.openai import BaseOpenAILLMService
from pipecat.services.openai import (
BaseOpenAILLMService,
OpenAIAssistantContextAggregator,
OpenAIContextAggregatorPair,
OpenAIUserContextAggregator,
)
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
@@ -38,6 +44,7 @@ try:
SpeechConfig,
SpeechRecognizer,
SpeechSynthesizer,
SpeechSynthesisOutputFormat,
)
from azure.cognitiveservices.speech.audio import (
AudioStreamFormat,
@@ -70,6 +77,33 @@ class AzureLLMService(BaseOpenAILLMService):
api_version=self._api_version,
)
@staticmethod
def create_context_aggregator(
context: OpenAILLMContext, *, assistant_expect_stripped_words: bool = True
) -> OpenAIContextAggregatorPair:
user = OpenAIUserContextAggregator(context)
assistant = OpenAIAssistantContextAggregator(
user, expect_stripped_words=assistant_expect_stripped_words
)
return OpenAIContextAggregatorPair(_user=user, _assistant=assistant)
def sample_rate_to_output_format(sample_rate: int) -> SpeechSynthesisOutputFormat:
match sample_rate:
case 8000:
return SpeechSynthesisOutputFormat.Raw8Khz16BitMonoPcm
case 16000:
return SpeechSynthesisOutputFormat.Raw16Khz16BitMonoPcm
case 22050:
return SpeechSynthesisOutputFormat.Raw22050Hz16BitMonoPcm
case 24000:
return SpeechSynthesisOutputFormat.Raw24Khz16BitMonoPcm
case 44100:
return SpeechSynthesisOutputFormat.Raw44100Hz16BitMonoPcm
case 48000:
return SpeechSynthesisOutputFormat.Raw48Khz16BitMonoPcm
return SpeechSynthesisOutputFormat.Raw16Khz16BitMonoPcm
class AzureTTSService(TTSService):
class InputParams(BaseModel):
@@ -88,13 +122,15 @@ class AzureTTSService(TTSService):
api_key: str,
region: str,
voice="en-US-SaraNeural",
sample_rate: int = 16000,
sample_rate: int = 24000,
params: InputParams = InputParams(),
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
speech_config = SpeechConfig(subscription=api_key, region=region)
speech_config.set_speech_synthesis_output_format(sample_rate_to_output_format(sample_rate))
self._speech_synthesizer = SpeechSynthesizer(speech_config=speech_config, audio_config=None)
self._settings = {
@@ -283,7 +319,7 @@ class AzureSTTService(STTService):
api_key: str,
region: str,
language=Language.EN_US,
sample_rate=16000,
sample_rate=24000,
channels=1,
**kwargs,
):

7
src/pipecat/services/cartesia.py
View File

@@ -80,7 +80,7 @@ class CartesiaTTSService(WordTTSService):
cartesia_version: str = "2024-06-10",
url: str = "wss://api.cartesia.ai/tts/websocket",
model: str = "sonic-english",
sample_rate: int = 16000,
sample_rate: int = 24000,
encoding: str = "pcm_s16le",
container: str = "raw",
params: InputParams = InputParams(),
@@ -99,6 +99,7 @@ class CartesiaTTSService(WordTTSService):
super().__init__(
aggregate_sentences=True,
push_text_frames=False,
sample_rate=sample_rate,
**kwargs,
)
@@ -298,13 +299,13 @@ class CartesiaHttpTTSService(TTSService):
voice_id: str,
model: str = "sonic-english",
base_url: str = "https://api.cartesia.ai",
sample_rate: int = 16000,
sample_rate: int = 24000,
encoding: str = "pcm_s16le",
container: str = "raw",
params: InputParams = InputParams(),
**kwargs,
):
super().__init__(**kwargs)
super().__init__(sample_rate=sample_rate, **kwargs)
self._api_key = api_key
self._settings = {

4
src/pipecat/services/deepgram.py
View File

@@ -51,11 +51,11 @@ class DeepgramTTSService(TTSService):
*,
api_key: str,
voice: str = "aura-helios-en",
sample_rate: int = 16000,
sample_rate: int = 24000,
encoding: str = "linear16",
**kwargs,
):
super().__init__(**kwargs)
super().__init__(sample_rate=sample_rate, **kwargs)
self._settings = {
"sample_rate": sample_rate,

2
src/pipecat/services/elevenlabs.py
View File

@@ -99,7 +99,7 @@ class ElevenLabsTTSService(WordTTSService):
voice_id: str,
model: str = "eleven_turbo_v2_5",
url: str = "wss://api.elevenlabs.io",
output_format: ElevenLabsOutputFormat = "pcm_16000",
output_format: ElevenLabsOutputFormat = "pcm_24000",
params: InputParams = InputParams(),
**kwargs,
):

4
src/pipecat/services/playht.py
View File

@@ -115,7 +115,7 @@ class PlayHTTTSService(TTSService):
user_id: str,
voice_url: str,
voice_engine: str = "PlayHT3.0-mini",
sample_rate: int = 16000,
sample_rate: int = 24000,
output_format: str = "wav",
params: InputParams = InputParams(),
**kwargs,
@@ -310,7 +310,7 @@ class PlayHTHttpTTSService(TTSService):
user_id: str,
voice_url: str,
voice_engine: str = "PlayHT3.0-mini",
sample_rate: int = 16000,
sample_rate: int = 24000,
params: InputParams = InputParams(),
**kwargs,
):

15
src/pipecat/services/xtts.py
View File

@@ -39,9 +39,10 @@ class XTTSService(TTSService):
language: Language,
base_url: str,
aiohttp_session: aiohttp.ClientSession,
sample_rate: int = 24000,
**kwargs,
):
super().__init__(**kwargs)
super().__init__(sample_rate=sample_rate, **kwargs)
self._settings = {
"language": self.language_to_service_language(language),
@@ -162,16 +163,18 @@ class XTTSService(TTSService):
# Remove processed data from buffer
buffer = buffer[48000:]
# Resample the audio from 24000 Hz to 16000 Hz
resampled_audio = resample_audio(bytes(process_data), 24000, 16000)
# Resample the audio from 24000 Hz
resampled_audio = resample_audio(
bytes(process_data), 24000, self._sample_rate
)
# Create the frame with the resampled audio
frame = TTSAudioRawFrame(resampled_audio, 16000, 1)
frame = TTSAudioRawFrame(resampled_audio, self._sample_rate, 1)
yield frame
# Process any remaining data in the buffer
if len(buffer) > 0:
resampled_audio = resample_audio(bytes(buffer), 24000, 16000)
frame = TTSAudioRawFrame(resampled_audio, 16000, 1)
resampled_audio = resample_audio(bytes(buffer), 24000, self._sample_rate)
frame = TTSAudioRawFrame(resampled_audio, self._sample_rate, 1)
yield frame
yield TTSStoppedFrame()

2
src/pipecat/transports/base_transport.py
View File

@@ -30,7 +30,7 @@ class TransportParams(BaseModel):
camera_out_color_format: str = "RGB"
audio_out_enabled: bool = False
audio_out_is_live: bool = False
audio_out_sample_rate: int = 16000
audio_out_sample_rate: int = 24000
audio_out_channels: int = 1
audio_out_bitrate: int = 96000
audio_in_enabled: bool = False

1
src/pipecat/transports/services/livekit.py
View File

@@ -11,6 +11,7 @@ from typing import Any, Awaitable, Callable, List
from pydantic import BaseModel
from pipecat.audio.utils import resample_audio
from pipecat.audio.vad.vad_analyzer import VADAnalyzer
from pipecat.frames.frames import (
AudioRawFrame,
CancelFrame,