introduce PipelineParams audio input/output sample rates

This commit is contained in:
Aleix Conchillo Flaqué
2025-02-04 12:22:41 -08:00
parent cc54255c41
commit ab45e481be
61 changed files with 570 additions and 402 deletions

View File

@@ -203,6 +203,8 @@ class GeminiMultimodalLiveLLMService(LLMService):
self._bot_audio_buffer = bytearray()
self._bot_text_buffer = ""
self._sample_rate = 24000
self._settings = {
"frequency_penalty": params.frequency_penalty,
"max_tokens": params.max_tokens,
@@ -521,7 +523,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
if self._audio_input_paused:
return
# Send all audio to Gemini
evt = events.AudioInputMessage.from_raw_audio(frame.audio)
evt = events.AudioInputMessage.from_raw_audio(frame.audio, frame.sample_rate)
await self.send_client_event(evt)
# Manage a buffer of audio to use for transcription
audio = frame.audio
@@ -650,7 +652,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
inline_data = part.inlineData
if not inline_data:
return
if inline_data.mimeType != "audio/pcm;rate=24000":
if inline_data.mimeType != f"audio/pcm;rate={self._sample_rate}":
logger.warning(f"Unrecognized server_content format {inline_data.mimeType}")
return
@@ -665,7 +667,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
self._bot_audio_buffer.extend(audio)
frame = TTSAudioRawFrame(
audio=audio,
sample_rate=24000,
sample_rate=self._sample_rate,
num_channels=1,
)
await self.push_frame(frame)