wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update TTS and transport output sample rate to 24000

Browse Source
This commit is contained in:
Aleix Conchillo Flaqué
2024-10-24 14:27:57 -07:00
parent d24c6185d8
commit 92a69e404f
16 changed files with 33 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGELOG.md
View File

@@ -13,6 +13,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
grained control of what media subscriptions you want for each participant in a
room.
### Changed
- Changed default output sample rate to 24000. This changes all TTS service to
output to 24000 and also the default output transport sample rate. This
improves audio quality at the cost of some extra bandwidth.
## [0.0.47] - 2024-10-22
### Added

2
examples/foundational/01b-livekit-audio.py
View File

@@ -81,7 +81,7 @@ async def main():
url=url,
token=token,
room_name=room_name,
params=LiveKitParams(audio_out_enabled=True, audio_out_sample_rate=16000),
params=LiveKitParams(audio_out_enabled=True),
)
tts = CartesiaTTSService(

1
examples/foundational/07e-interruptible-playht.py
View File

@@ -40,7 +40,6 @@ async def main():
"Respond bot",
DailyParams(
audio_out_enabled=True,
audio_out_sample_rate=16000,
transcription_enabled=True,
vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(),

1
examples/foundational/07f-interruptible-azure.py
View File

@@ -41,7 +41,6 @@ async def main():
"Respond bot",
DailyParams(
audio_out_enabled=True,
audio_out_sample_rate=16000,
vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,

1
examples/foundational/09-mirror.py
View File

@@ -63,6 +63,7 @@ async def main():
"Test",
DailyParams(
audio_in_enabled=True,
audio_in_sample_rate=24000,
audio_out_enabled=True,
camera_out_enabled=True,
camera_out_is_live=True,

2
examples/foundational/09a-local-mirror.py
View File

@@ -65,7 +65,7 @@ async def main():
tk_root.title("Local Mirror")
daily_transport = DailyTransport(
room_url, token, "Test", DailyParams(audio_in_enabled=True)
room_url, token, "Test", DailyParams(audio_in_enabled=True, audio_in_sample_rate=24000)
)
tk_transport = TkLocalTransport(

3
examples/foundational/12c-describe-video-anthropic.py
View File

@@ -78,9 +78,6 @@ async def main():
tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady
params=CartesiaTTSService.InputParams(
sample_rate=16000,
),
)
@transport.event_handler("on_first_participant_joined")

2
src/pipecat/audio/vad/silero.py
View File

@@ -52,7 +52,7 @@ class SileroOnnxModel:
if sr not in self.sample_rates:
raise ValueError(
f"Supported sampling rates: {self.sample_rates} (or multiply of 16000)"
f"Supported sampling rates: {self.sample_rates} (or multiple of 16000)"
)
if sr / np.shape(x)[1] > 31.25:
raise ValueError("Input audio chunk is too short")

4
src/pipecat/services/ai_services.py
View File

@@ -205,7 +205,7 @@ class TTSService(AIService):
# if push_stop_frames is True, wait for this idle period before pushing TTSStoppedFrame
stop_frame_timeout_s: float = 1.0,
# TTS output sample rate
sample_rate: int = 16000,
sample_rate: int = 24000,
text_filter: Optional[BaseTextFilter] = None,
**kwargs,
):
@@ -514,7 +514,7 @@ class SegmentedSTTService(STTService):
min_volume: float = 0.6,
max_silence_secs: float = 0.3,
max_buffer_secs: float = 1.5,
sample_rate: int = 16000,
sample_rate: int = 24000,
num_channels: int = 1,
**kwargs,
):

4
src/pipecat/services/azure.py
View File

@@ -88,7 +88,7 @@ class AzureTTSService(TTSService):
api_key: str,
region: str,
voice="en-US-SaraNeural",
sample_rate: int = 16000,
sample_rate: int = 24000,
params: InputParams = InputParams(),
**kwargs,
):
@@ -283,7 +283,7 @@ class AzureSTTService(STTService):
api_key: str,
region: str,
language=Language.EN_US,
sample_rate=16000,
sample_rate=24000,
channels=1,
**kwargs,
):

7
src/pipecat/services/cartesia.py
View File

@@ -80,7 +80,7 @@ class CartesiaTTSService(WordTTSService):
cartesia_version: str = "2024-06-10",
url: str = "wss://api.cartesia.ai/tts/websocket",
model: str = "sonic-english",
sample_rate: int = 16000,
sample_rate: int = 24000,
encoding: str = "pcm_s16le",
container: str = "raw",
params: InputParams = InputParams(),
@@ -99,6 +99,7 @@ class CartesiaTTSService(WordTTSService):
super().__init__(
aggregate_sentences=True,
push_text_frames=False,
sample_rate=sample_rate,
**kwargs,
)
@@ -298,13 +299,13 @@ class CartesiaHttpTTSService(TTSService):
voice_id: str,
model: str = "sonic-english",
base_url: str = "https://api.cartesia.ai",
sample_rate: int = 16000,
sample_rate: int = 24000,
encoding: str = "pcm_s16le",
container: str = "raw",
params: InputParams = InputParams(),
**kwargs,
):
super().__init__(**kwargs)
super().__init__(sample_rate=sample_rate, **kwargs)
self._api_key = api_key
self._settings = {

4
src/pipecat/services/deepgram.py
View File

@@ -51,11 +51,11 @@ class DeepgramTTSService(TTSService):
*,
api_key: str,
voice: str = "aura-helios-en",
sample_rate: int = 16000,
sample_rate: int = 24000,
encoding: str = "linear16",
**kwargs,
):
super().__init__(**kwargs)
super().__init__(sample_rate=sample_rate, **kwargs)
self._settings = {
"sample_rate": sample_rate,

2
src/pipecat/services/elevenlabs.py
View File

@@ -99,7 +99,7 @@ class ElevenLabsTTSService(WordTTSService):
voice_id: str,
model: str = "eleven_turbo_v2_5",
url: str = "wss://api.elevenlabs.io",
output_format: ElevenLabsOutputFormat = "pcm_16000",
output_format: ElevenLabsOutputFormat = "pcm_24000",
params: InputParams = InputParams(),
**kwargs,
):

4
src/pipecat/services/playht.py
View File

@@ -115,7 +115,7 @@ class PlayHTTTSService(TTSService):
user_id: str,
voice_url: str,
voice_engine: str = "PlayHT3.0-mini",
sample_rate: int = 16000,
sample_rate: int = 24000,
output_format: str = "wav",
params: InputParams = InputParams(),
**kwargs,
@@ -310,7 +310,7 @@ class PlayHTHttpTTSService(TTSService):
user_id: str,
voice_url: str,
voice_engine: str = "PlayHT3.0-mini",
sample_rate: int = 16000,
sample_rate: int = 24000,
params: InputParams = InputParams(),
**kwargs,
):

15
src/pipecat/services/xtts.py
View File

@@ -39,9 +39,10 @@ class XTTSService(TTSService):
language: Language,
base_url: str,
aiohttp_session: aiohttp.ClientSession,
sample_rate: int = 24000,
**kwargs,
):
super().__init__(**kwargs)
super().__init__(sample_rate=sample_rate, **kwargs)
self._settings = {
"language": self.language_to_service_language(language),
@@ -162,16 +163,18 @@ class XTTSService(TTSService):
# Remove processed data from buffer
buffer = buffer[48000:]
# Resample the audio from 24000 Hz to 16000 Hz
resampled_audio = resample_audio(bytes(process_data), 24000, 16000)
# Resample the audio from 24000 Hz
resampled_audio = resample_audio(
bytes(process_data), 24000, self._sample_rate
)
# Create the frame with the resampled audio
frame = TTSAudioRawFrame(resampled_audio, 16000, 1)
frame = TTSAudioRawFrame(resampled_audio, self._sample_rate, 1)
yield frame
# Process any remaining data in the buffer
if len(buffer) > 0:
resampled_audio = resample_audio(bytes(buffer), 24000, 16000)
frame = TTSAudioRawFrame(resampled_audio, 16000, 1)
resampled_audio = resample_audio(bytes(buffer), 24000, self._sample_rate)
frame = TTSAudioRawFrame(resampled_audio, self._sample_rate, 1)
yield frame
yield TTSStoppedFrame()

2
src/pipecat/transports/base_transport.py
View File

@@ -30,7 +30,7 @@ class TransportParams(BaseModel):
camera_out_color_format: str = "RGB"
audio_out_enabled: bool = False
audio_out_is_live: bool = False
audio_out_sample_rate: int = 16000
audio_out_sample_rate: int = 24000
audio_out_channels: int = 1
audio_out_bitrate: int = 96000
audio_in_enabled: bool = False