From 133d7ee33a4dcf934b4b7f5dd1fc2a2125a9bc76 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Mon, 12 May 2025 10:16:32 -0300 Subject: [PATCH 1/3] Fixing the default audio source for capture_participant_audio --- src/pipecat/transports/services/daily.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index f1a514d0e..2b84c22ae 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -702,6 +702,8 @@ class DailyTransportClient(EventHandler): self._audio_renderers.setdefault(participant_id, {})[audio_source] = callback + logger.info(f"Starting to capture audio from participant {participant_id} to {audio_source}") + self._client.set_audio_renderer( participant_id, self._audio_data_received, @@ -1022,7 +1024,7 @@ class DailyInputTransport(BaseInputTransport): async def capture_participant_audio( self, participant_id: str, - audio_source: str = "camera", + audio_source: str = "microphone", ): await self._client.capture_participant_audio( participant_id, self._on_participant_audio_data, audio_source From 983199a6cdf1beadcc4594d4a3ddbb852684fa31 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Mon, 12 May 2025 10:18:43 -0300 Subject: [PATCH 2/3] New example capturing the audio from the participant using the custom audio source. --- .../04c-transports-daily-audio-source.py | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 examples/foundational/04c-transports-daily-audio-source.py diff --git a/examples/foundational/04c-transports-daily-audio-source.py b/examples/foundational/04c-transports-daily-audio-source.py new file mode 100644 index 000000000..aa62a543f --- /dev/null +++ b/examples/foundational/04c-transports-daily-audio-source.py @@ -0,0 +1,112 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from daily_runner import configure +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +from pipecat.services.deepgram.stt import DeepgramSTTService, Language, LiveOptions + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_in_enabled=True, + audio_in_passthrough=False, + audio_out_enabled=True, + audio_out_sample_rate=16000, + transcription_enabled=False, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + stt = DeepgramSTTService( + api_key=os.getenv("DEEPGRAM_API_KEY"), + live_options=LiveOptions(language=Language.EN), + ) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_audio(participant["id"]) + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + await task.cancel() + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) From d22dbb1a6d1ab5c0c839479cd57ce4d4de50d85c Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Mon, 12 May 2025 10:36:21 -0300 Subject: [PATCH 3/3] Fixing ruff format. --- examples/foundational/04c-transports-daily-audio-source.py | 3 +-- src/pipecat/transports/services/daily.py | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/foundational/04c-transports-daily-audio-source.py b/examples/foundational/04c-transports-daily-audio-source.py index aa62a543f..00cb8a603 100644 --- a/examples/foundational/04c-transports-daily-audio-source.py +++ b/examples/foundational/04c-transports-daily-audio-source.py @@ -19,11 +19,10 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService, Language, LiveOptions from pipecat.services.openai.llm import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from pipecat.services.deepgram.stt import DeepgramSTTService, Language, LiveOptions - load_dotenv(override=True) logger.remove(0) diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index 2b84c22ae..4528662a6 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -702,7 +702,9 @@ class DailyTransportClient(EventHandler): self._audio_renderers.setdefault(participant_id, {})[audio_source] = callback - logger.info(f"Starting to capture audio from participant {participant_id} to {audio_source}") + logger.info( + f"Starting to capture audio from participant {participant_id} to {audio_source}" + ) self._client.set_audio_renderer( participant_id,