diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index b806efad4..bd6961dd5 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -1,4 +1,4 @@
-name: test
+name: tests
on:
workflow_dispatch:
@@ -49,4 +49,4 @@ jobs:
- name: Test with pytest
run: |
source .venv/bin/activate
- pytest --ignore-glob="*to_be_updated*" --ignore-glob=*pipeline_source* src tests
+ pytest
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 37b927ecd..2162047cd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,12 +5,205 @@ All notable changes to **Pipecat** will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [0.0.55] - 2025-02-05
+
+### Added
+
+- Added a new `start_metadata` field to `PipelineParams`. The provided metadata
+ will be set to the initial `StartFrame` being pushed from the `PipelineTask`.
+
+- Added new fields to `PipelineParams` to control audio input and output sample
+ rates for the whole pipeline. This allows controlling sample rates from a
+ single place instead of having to specify sample rates in each
+ service. Setting a sample rate to a service is still possible and will
+ override the value from `PipelineParams`.
+
+- Introduce audio resamplers (`BaseAudioResampler`). This is just a base class
+ to implement audio resamplers. Currently, two implementations are provided
+ `SOXRAudioResampler` and `ResampyResampler`. A new
+ `create_default_resampler()` has been added (replacing the now deprecated
+ `resample_audio()`).
+
+- It is now possible to specify the asyncio event loop that a `PipelineTask` and
+ all the processors should run on by passing it as a new argument to the
+ `PipelineRunner`. This could allow running pipelines in multiple threads each
+ one with its own event loop.
+
+- Added a new `utils.TaskManager`. Instead of a global task manager we now have
+ a task manager per `PipelineTask`. In the previous version the task manager
+ was global, so running multiple simultaneous `PipelineTask`s could result in
+ dangling task warnings which were not actually true. In order, for all the
+ processors to know about the task manager, we pass it through the
+ `StartFrame`. This means that processors should create tasks when they receive
+ a `StartFrame` but not before (because they don't have a task manager yet).
+
+- Added `TelnyxFrameSerializer` to support Telnyx calls. A full running example
+ has also been added to `examples/telnyx-chatbot`.
+
+- Allow pushing silence audio frames before `TTSStoppedFrame`. This might be
+ useful for testing purposes, for example, passing bot audio to an STT service
+ which usually needs additional audio data to detect the utterance stopped.
+
+- `TwilioSerializer` now supports transport message frames. With this we can
+ create Twilio emulators.
+
+- Added a new transport: `WebsocketClientTransport`.
+
+- Added a `metadata` field to `Frame` which makes it possible to pass custom
+ data to all frames.
+
+- Added `test/utils.py` inside of pipecat package.
+
+### Changed
+
+- `GatedOpenAILLMContextAggregator` now require keyword arguments. Also, a new
+ `start_open` argument has been added to set the initial state of the gate.
+
+- Added `organization` and `project` level authentication to
+ `OpenAILLMService`.
+
+- Improved the language checking logic in `ElevenLabsTTSService` and
+ `ElevenLabsHttpTTSService` to properly handle language codes based on model
+ compatibility, with appropriate warnings when language codes cannot be
+ applied.
+
+- Updated `GoogleLLMContext` to support pushing `LLMMessagesUpdateFrame`s that
+ contain a combination of function calls, function call responses, system
+ messages, or just messages.
+
+- `InputDTMFFrame` is now based on `DTMFFrame`. There's also a new
+ `OutputDTMFFrame` frame.
+
+### Deprecated
+
+- `resample_audio()` is now deprecated, use `create_default_resampler()`
+ instead.
+
+### Removed
+
+- `AudioBufferProcessor.reset_audio_buffers()` has been removed, use
+ `AudioBufferProcessor.start_recording()` and
+ ``AudioBufferProcessor.stop_recording()` instead.
+
+### Fixed
+
+- Fixed a `AudioBufferProcessor` that would cause crackling in some recordings.
+
+- Fixed an issue in `AudioBufferProcessor` where user callback would not be
+ called on task cancellation.
+
+- Fixed an issue in `AudioBufferProcessor` that would cause wrong silence
+ padding in some cases.
+
+- Fixed an issue where `ElevenLabsTTSService` messages would return a 1009
+ websocket error by increasing the max message size limit to 16MB.
+
+- Fixed a `DailyTransport` issue that would cause events to be triggered before
+ join finished.
+
+- Fixed a `PipelineTask` issue that was preventing processors to be cleaned up
+ after cancelling the task.
+
+- Fixed an issue where queuing a `CancelFrame` to a pipeline task would not
+ cause the task to finish. However, using `PipelineTask.cancel()` is still the
+ recommended way to cancel a task.
+
+### Other
+
+- Improved Unit Test `run_test()` to use `PipelineTask` and
+ `PipelineRunner`. There's now also some control around `StartFrame` and
+ `EndFrame`. The `EndTaskFrame` has been removed since it doesn't seem
+ necessary with this new approach.
+
+- Updated `twilio-chatbot` with a few new features: use 8000 sample rate and
+ avoid resampling, a new client useful for stress testing and testing locally
+ without the need to make phone calls. Also, added audio recording on both the
+ client and the server to make sure the audio sounds good.
+
+- Updated examples to use `task.cancel()` to immediately exit the example when a
+ participant leaves or disconnects, instead of pushing an `EndFrame`. Pushing
+ an `EndFrame` causes the bot to run through everything that is internally
+ queued (which could take some seconds). Note that using `task.cancel()` might
+ not always be the best option and pushing an `EndFrame` could still be
+ desirable to make sure all the pipeline is flushed.
+
+## [0.0.54] - 2025-01-27
+
+### Added
+
+- In order to create tasks in Pipecat frame processors it is now recommended to
+ use `FrameProcessor.create_task()` (which uses the new
+ `utils.asyncio.create_task()`). It takes care of uncaught exceptions, task
+ cancellation handling and task management. To cancel or wait for a task there
+ is `FrameProcessor.cancel_task()` and `FrameProcessor.wait_for_task()`. All of
+ Pipecat processors have been updated accordingly. Also, when a pipeline runner
+ finishes, a warning about dangling tasks might appear, which indicates if any
+ of the created tasks was never cancelled or awaited for (using these new
+ functions).
+
+- It is now possible to specify the period of the `PipelineTask` heartbeat
+ frames with `heartbeats_period_secs`.
+
+- Added `DailyMeetingTokenProperties` and `DailyMeetingTokenParams` Pydantic models
+ for meeting token creation in `get_token` method of `DailyRESTHelper`.
+
+- Added `enable_recording` and `geo` parameters to `DailyRoomProperties`.
+
+- Added `RecordingsBucketConfig` to `DailyRoomProperties` to upload recordings to a custom AWS bucket.
+
+### Changed
+
+- Enhanced `UserIdleProcessor` with retry functionality and control over idle
+ monitoring via new callback signature `(processor, retry_count) -> bool`.
+ Updated the `17-detect-user-idle.py` to show how to use the `retry_count`.
+
+- Add defensive error handling for `OpenAIRealtimeBetaLLMService`'s audio
+ truncation. Audio truncation errors during interruptions now log a warning
+ and allow the session to continue instead of throwing an exception.
+
+- Modified `TranscriptProcessor` to use TTS text frames for more accurate assistant
+ transcripts. Assistant messages are now aggregated based on bot speaking boundaries
+ rather than LLM context, providing better handling of interruptions and partial
+ utterances.
+
+- Updated foundational examples `28a-transcription-processor-openai.py`,
+ `28b-transcript-processor-anthropic.py`, and
+ `28c-transcription-processor-gemini.py` to use the updated
+ `TranscriptProcessor`.
+
+### Fixed
+
+- Fixed an `GeminiMultimodalLiveLLMService` issue that was preventing the user
+ to push initial LLM assistant messages (using `LLMMessagesAppendFrame`).
+
+- Added missing `FrameProcessor.cleanup()` calls to `Pipeline`,
+ `ParallelPipeline` and `UserIdleProcessor`.
+
+- Fixed a type error when using `voice_settings` in `ElevenLabsHttpTTSService`.
+
+- Fixed an issue where `OpenAIRealtimeBetaLLMService` function calling resulted
+ in an error.
+
+- Fixed an issue in `AudioBufferProcessor` where the last audio buffer was not
+ being processed, in cases where the `_user_audio_buffer` was smaller than the
+ buffer size.
+
+### Performance
+
+- Replaced audio resampling library `resampy` with `soxr`. Resampling a 2:21s
+ audio file from 24KHz to 16KHz took 1.41s with `resampy` and 0.031s with
+ `soxr` with similar audio quality.
+
+### Other
+
+- Added initial unit test infrastructure.
+
## [0.0.53] - 2025-01-18
### Added
-- Added `ElevenLabsHttpTTSService` and the
- `07d-interruptible-elevenlabs-http.py` foundational example.
+- Added `ElevenLabsHttpTTSService` which uses EleveLabs' HTTP API instead of the
+ websocket one.
- Introduced pipeline frame observers. Observers can view all the frames that go
through the pipeline without the need to inject processors in the
@@ -1381,6 +1574,9 @@ async def on_connected(processor):
### Changed
+- `FrameSerializer.serialize()` and `FrameSerializer.deserialize()` are now
+ `async`.
+
- `Filter` has been renamed to `FrameFilter` and it's now under
`processors/filters`.
diff --git a/README.md b/README.md
index 52c6f831b..5a915041d 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
AI Chatbot
+
+
+
+
+
+
+ Status: Disconnected
+
+
+ Connect
+ Disconnect
+
+
+
+
+
+
+
-
-
AI Chatbot
+
+
+
+
+
+
+ Status: Disconnected
+
+
+ Connect
+ Disconnect
+
+
+
+
+
+
+
+
+
= ({ handleLeave }) => {
setTimeout(() => daily.setLocalAudio(true), 500);
setStoryState("user");
} else {
- daily.setLocalAudio(false);
+ // Uncomment the next line to mute the mic while the
+ // assistant it talking. Leave it commented to allow for interruptions
+ // daily.setLocalAudio(false);
setStoryState("assistant");
}
},
@@ -58,7 +60,7 @@ const Story: React.FC = ({ handleLeave }) => {
{participantIds.length >= 1 ? (
@@ -71,7 +73,7 @@ const Story: React.FC = ({ handleLeave }) => {
)}
([]);
+ const timeoutRef = useRef();
+
+ const resetTimeout = () => {
+ if (timeoutRef.current) {
+ clearTimeout(timeoutRef.current);
+ }
+ timeoutRef.current = setTimeout(() => {
+ setTranscription([]);
+ }, 5000);
+ };
+
+ useEffect(() => {
+ return () => {
+ if (timeoutRef.current) {
+ clearTimeout(timeoutRef.current);
+ }
+ };
+ }, []);
useAppMessage({
onAppMessage: (e: DailyEventObjectAppMessage) => {
if (e.fromId && e.fromId === "transcription") {
if (e.data.user_id === "" && e.data.is_final) {
setTranscription((t) => [...t, ...e.data.text.split(" ")]);
+ resetTimeout();
}
}
},
diff --git a/examples/storytelling-chatbot/frontend/components/VideoTile/VideoTile.module.css b/examples/storytelling-chatbot/frontend/components/VideoTile/VideoTile.module.css
index 7158ca10d..975c8a5bb 100644
--- a/examples/storytelling-chatbot/frontend/components/VideoTile/VideoTile.module.css
+++ b/examples/storytelling-chatbot/frontend/components/VideoTile/VideoTile.module.css
@@ -6,8 +6,8 @@
.videoTile{
@apply bg-gray-950;
- width: 560px;
- height: 560px;
+ width: 760px;
+ height: 760px;
mask-image: url('/alpha-mask.gif');
mask-size: cover;
mask-repeat: no-repeat;
diff --git a/examples/storytelling-chatbot/requirements.txt b/examples/storytelling-chatbot/requirements.txt
index 0cebe6edb..c8cd82646 100644
--- a/examples/storytelling-chatbot/requirements.txt
+++ b/examples/storytelling-chatbot/requirements.txt
@@ -2,4 +2,5 @@ async_timeout
fastapi
uvicorn
python-dotenv
-pipecat-ai[daily,elevenlabs,openai,fal]
+-e "../..[daily,silero,openai,fal,cartesia,google]"
+-e "../../../python-genai"
diff --git a/examples/storytelling-chatbot/src/assets/book1.png b/examples/storytelling-chatbot/src/assets/book1.png
index 4148dbaa7..9c1b11329 100644
Binary files a/examples/storytelling-chatbot/src/assets/book1.png and b/examples/storytelling-chatbot/src/assets/book1.png differ
diff --git a/examples/storytelling-chatbot/src/assets/book2.png b/examples/storytelling-chatbot/src/assets/book2.png
index c7dee447b..173947446 100644
Binary files a/examples/storytelling-chatbot/src/assets/book2.png and b/examples/storytelling-chatbot/src/assets/book2.png differ
diff --git a/examples/storytelling-chatbot/src/bot.py b/examples/storytelling-chatbot/src/bot.py
index bc4eb62cc..05f8974bb 100644
--- a/examples/storytelling-chatbot/src/bot.py
+++ b/examples/storytelling-chatbot/src/bot.py
@@ -13,17 +13,18 @@ import aiohttp
from dotenv import load_dotenv
from loguru import logger
from processors import StoryImageProcessor, StoryProcessor
-from prompts import CUE_USER_TURN, LLM_BASE_PROMPT, LLM_INTRO_PROMPT
+from prompts import CUE_USER_TURN, LLM_BASE_PROMPT
from utils.helpers import load_images, load_sounds
-from pipecat.frames.frames import EndFrame, LLMMessagesFrame, StopTaskFrame
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.frames.frames import EndFrame
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineTask
+from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.elevenlabs import ElevenLabsTTSService
from pipecat.services.fal import FalImageGenService
-from pipecat.services.openai import OpenAILLMService
+from pipecat.services.google import GoogleLLMService
from pipecat.transports.services.daily import (
DailyParams,
DailyTransport,
@@ -50,9 +51,10 @@ async def main(room_url, token=None):
DailyParams(
audio_out_enabled=True,
camera_out_enabled=True,
- camera_out_width=768,
- camera_out_height=768,
+ camera_out_width=1024,
+ camera_out_height=1024,
transcription_enabled=True,
+ vad_analyzer=SileroVADAnalyzer(),
vad_enabled=True,
),
)
@@ -61,23 +63,13 @@ async def main(room_url, token=None):
# -------------- Services --------------- #
- llm_service = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
+ llm_service = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"))
tts_service = ElevenLabsTTSService(
- api_key=os.getenv("ELEVENLABS_API_KEY"),
- voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+ api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id=os.getenv("ELEVENLABS_VOICE_ID")
)
- fal_service_params = FalImageGenService.InputParams(
- image_size={"width": 768, "height": 768}
- )
-
- fal_service = FalImageGenService(
- aiohttp_session=session,
- model="fal-ai/fast-lightning-sdxl",
- params=fal_service_params,
- key=os.getenv("FAL_KEY"),
- )
+ image_gen = GoogleImageGenService(api_key=os.getenv("GOOGLE_API_KEY"))
# --------------- Setup ----------------- #
@@ -91,41 +83,13 @@ async def main(room_url, token=None):
# -------------- Processors ------------- #
story_processor = StoryProcessor(message_history, story_pages)
- image_processor = StoryImageProcessor(fal_service)
+ image_processor = StoryImageProcessor(image_gen)
# -------------- Story Loop ------------- #
runner = PipelineRunner()
- # The intro pipeline is used to start
- # the story (as per LLM_INTRO_PROMPT)
- intro_pipeline = Pipeline([llm_service, tts_service, transport.output()])
-
- intro_task = PipelineTask(intro_pipeline)
-
logger.debug("Waiting for participant...")
-
- @transport.event_handler("on_first_participant_joined")
- async def on_first_participant_joined(transport, participant):
- logger.debug("Participant joined, storytime commence!")
- await transport.capture_participant_transcription(participant["id"])
- await intro_task.queue_frames(
- [
- images["book1"],
- LLMMessagesFrame([LLM_INTRO_PROMPT]),
- DailyTransportMessageFrame(CUE_USER_TURN),
- sounds["listening"],
- images["book2"],
- StopTaskFrame(),
- ]
- )
-
- # We run the intro pipeline. This will start the transport. The intro
- # task will exit after StopTaskFrame is processed.
- await runner.run(intro_task)
-
- # The main story pipeline is used to continue the story based on user
- # input.
main_pipeline = Pipeline(
[
transport.input(),
@@ -139,16 +103,38 @@ async def main(room_url, token=None):
]
)
- main_task = PipelineTask(main_pipeline)
+ main_task = PipelineTask(
+ main_pipeline,
+ PipelineParams(
+ allow_interruptions=True,
+ enable_metrics=True,
+ enable_usage_metrics=True,
+ ),
+ )
+
+ @transport.event_handler("on_first_participant_joined")
+ async def on_first_participant_joined(transport, participant):
+ logger.debug("Participant joined, storytime commence!")
+ await transport.capture_participant_transcription(participant["id"])
+ await main_task.queue_frames(
+ [
+ images["book1"],
+ context_aggregator.user().get_context_frame(),
+ DailyTransportMessageFrame(CUE_USER_TURN),
+ # sounds["listening"],
+ images["book2"],
+ ]
+ )
@transport.event_handler("on_participant_left")
async def on_participant_left(transport, participant, reason):
- await intro_task.queue_frame(EndFrame())
- await main_task.queue_frame(EndFrame())
+ await main_task.cancel()
@transport.event_handler("on_call_state_updated")
async def on_call_state_updated(transport, state):
if state == "left":
+ # Here we don't want to cancel, we just want to finish sending
+ # whatever is queued, so we use an EndFrame().
await main_task.queue_frame(EndFrame())
await runner.run(main_task)
diff --git a/examples/storytelling-chatbot/src/bot_runner.py b/examples/storytelling-chatbot/src/bot_runner.py
index 97a60b72e..591b2d598 100644
--- a/examples/storytelling-chatbot/src/bot_runner.py
+++ b/examples/storytelling-chatbot/src/bot_runner.py
@@ -114,7 +114,7 @@ async def start_bot(request: Request) -> JSONResponse:
else:
try:
subprocess.Popen(
- [f"python3 -m bot -u {room.url} -t {token}"],
+ [f"python -m bot -u {room.url} -t {token}"],
shell=True,
bufsize=1,
cwd=os.path.dirname(os.path.abspath(__file__)),
@@ -175,7 +175,7 @@ async def virtualize_bot(room_url: str, token: str):
image = data[0]["config"]["image"]
# Machine configuration
- cmd = f"python3 src/bot.py -u {room_url} -t {token}"
+ cmd = f"python src/bot.py -u {room_url} -t {token}"
cmd = cmd.split()
worker_props = {
"config": {
@@ -215,9 +215,8 @@ async def virtualize_bot(room_url: str, token: str):
if __name__ == "__main__":
# Check environment variables
required_env_vars = [
- "OPENAI_API_KEY",
+ "GOOGLE_API_KEY",
"DAILY_API_KEY",
- "FAL_KEY",
"ELEVENLABS_VOICE_ID",
"ELEVENLABS_API_KEY",
]
diff --git a/examples/storytelling-chatbot/src/processors.py b/examples/storytelling-chatbot/src/processors.py
index 096efd577..62a8fcee4 100644
--- a/examples/storytelling-chatbot/src/processors.py
+++ b/examples/storytelling-chatbot/src/processors.py
@@ -1,16 +1,27 @@
+import os
import re
+import google.ai.generativelanguage as glm
from async_timeout import timeout
-from prompts import CUE_ASSISTANT_TURN, CUE_USER_TURN, IMAGE_GEN_PROMPT
+from loguru import logger
+from prompts import (
+ CUE_ASSISTANT_TURN,
+ CUE_USER_TURN,
+ FIRST_IMAGE_PROMPT,
+ IMAGE_GEN_PROMPT,
+ NEXT_IMAGE_PROMPT,
+)
from utils.helpers import load_sounds
from pipecat.frames.frames import (
Frame,
LLMFullResponseEndFrame,
+ LLMMessagesFrame,
TextFrame,
UserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.services.google import GoogleImageGenService, GoogleLLMContext, GoogleLLMService
from pipecat.transports.services.daily import DailyTransportMessageFrame
sounds = load_sounds(["talking.wav", "listening.wav", "ding.wav"])
@@ -37,39 +48,69 @@ class StoryPromptFrame(TextFrame):
class StoryImageProcessor(FrameProcessor):
- """
- Processor for image prompt frames that will be sent to the FAL service.
+ """Processor for image prompt frames that will be sent to the FAL service.
This processor is responsible for consuming frames of type `StoryImageFrame`.
It processes them by passing it to the FAL service.
The processed frames are then yielded back.
Attributes:
- _fal_service (FALService): The FAL service, generates the images (fast fast!).
+ _image_gen_service: The FAL service, generates the images (fast fast!).
"""
- def __init__(self, fal_service):
+ def __init__(self, image_gen_service):
super().__init__()
- self._fal_service = fal_service
+ self._image_gen_service = image_gen_service
+ # Create a new LLM service to use a different system prompt, etc
+ self._llm_service = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"))
+
+ self.pages = []
+ self.image_descriptions = []
+
+ def can_generate_metrics(self) -> bool:
+ return True
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
- if isinstance(frame, StoryImageFrame):
+ if isinstance(frame, StoryPageFrame):
+ # Special syntax for the first page
+ if self.pages == []:
+ prompt = FIRST_IMAGE_PROMPT % frame.text
+ else:
+ prompt = NEXT_IMAGE_PROMPT % (
+ " ".join(self.pages),
+ "; ".join(self.image_descriptions),
+ frame.text,
+ )
+
+ await self.start_ttfb_metrics()
+ # TODO: This is coupled to google implementation now
+ txt = glm.Content(role="user", parts=[glm.Part(text=prompt)])
+ llm_response = await self._llm_service._client.generate_content_async(
+ contents=[txt], stream=False
+ )
+ image_description = llm_response.text
+ self.pages.append(frame.text)
+ self.image_descriptions.append(image_description)
try:
- async with timeout(7):
- async for i in self._fal_service.run_image_gen(IMAGE_GEN_PROMPT % frame.text):
+ async with timeout(15):
+ async for i in self._image_gen_service.run_image_gen(
+ IMAGE_GEN_PROMPT % image_description
+ ):
await self.push_frame(i)
except TimeoutError:
+ logger.debug("Image gen timeout")
pass
- pass
+ await self.stop_ttfb_metrics()
+ # Push the StoryPageFrame so it gets TTS
+ await self.push_frame(frame)
else:
await self.push_frame(frame)
class StoryProcessor(FrameProcessor):
- """
- Primary frame processor. It takes the frames generated by the LLM
+ """Primary frame processor. It takes the frames generated by the LLM
and processes them into image prompts and story pages (sentences).
For a clearer picture of how this works, reference prompts.py
@@ -97,44 +138,11 @@ class StoryProcessor(FrameProcessor):
await self.push_frame(sounds["talking"])
elif isinstance(frame, TextFrame):
- # We want to look for sentence breaks in the text
- # but since TextFrames are streamed from the LLM
- # we need to keep a buffer of the text we've seen so far
- self._text += frame.text
-
- # IMAGE PROMPT
- # Looking for: < [image prompt] > in the LLM response
- # We prompted our LLM to add an image prompt in the response
- # so we use regex matching to find it and yield a StoryImageFrame
- if re.search(r"<.*?>", self._text):
- if not re.search(r"<.*?>.*?>", self._text):
- # Pass any frames until we have a closing bracket
- # otherwise the image prompt will be passed to TTS
- pass
- # Extract the image prompt from the text using regex
- image_prompt = re.search(r"<(.*?)>", self._text).group(1)
- # Remove the image prompt from the text
- self._text = re.sub(r"<.*?>", "", self._text, count=1)
- # Process the image prompt frame
- await self.push_frame(StoryImageFrame(image_prompt))
-
- # STORY PAGE
- # Looking for: [break] in the LLM response
- # We prompted our LLM to add a [break] after each sentence
- # so we use regex matching to find it in the LLM response
- if re.search(r".*\[[bB]reak\].*", self._text):
- # Remove the [break] token from the text
- # so it isn't spoken out loud by the TTS
- self._text = re.sub(r"\[[bB]reak\]", "", self._text, flags=re.IGNORECASE)
- self._text = self._text.replace("\n", " ")
- if len(self._text) > 2:
- # Append the sentence to the story
- self._story.append(self._text)
- await self.push_frame(StoryPageFrame(self._text))
- # Assert that it's the LLMs turn, until we're finished
- await self.push_frame(DailyTransportMessageFrame(CUE_ASSISTANT_TURN))
- # Clear the buffer
- self._text = ""
+ # Add new text to the buffer
+ # (character replace hack to fix TTS sequencing)
+ self._text += frame.text.replace(";", "—")
+ # Process any complete patterns in the order they appear
+ await self.process_text_content()
# End of a full LLM response
# Driven by the prompt, the LLM should have asked the user for input
@@ -150,3 +158,38 @@ class StoryProcessor(FrameProcessor):
# Anything that is not a TextFrame pass through
else:
await self.push_frame(frame)
+
+ async def process_text_content(self):
+ """Process text content in order of appearance, handling both image prompts and story breaks."""
+ while True:
+ # Find the first occurrence of each pattern
+ image_match = re.search(r"<(.*?)>", self._text)
+ break_match = re.search(r"\[[bB]reak\]", self._text)
+
+ # If neither pattern is found, we're done processing
+ if not image_match and not break_match:
+ break
+
+ # Find which pattern comes first in the text
+ image_pos = image_match.start() if image_match else float("inf")
+ break_pos = break_match.start() if break_match else float("inf")
+
+ if image_pos < break_pos:
+ # Process image prompt first
+ image_prompt = image_match.group(1)
+ # Remove the image prompt from the text
+ self._text = self._text[: image_match.start()] + self._text[image_match.end() :]
+ await self.push_frame(StoryImageFrame(image_prompt))
+ else:
+ # Process story break first
+ parts = re.split(r"\[[bB]reak\]", self._text, flags=re.IGNORECASE, maxsplit=1)
+ before_break = parts[0].replace("\n", " ").strip()
+
+ if len(before_break) > 2:
+ self._story.append(before_break)
+ await self.push_frame(StoryPageFrame(before_break))
+ # await self.push_frame(sounds["ding"])
+ await self.push_frame(DailyTransportMessageFrame(CUE_ASSISTANT_TURN))
+
+ # Keep the remainder (if any) in the buffer
+ self._text = parts[1].strip() if len(parts) > 1 else ""
diff --git a/examples/storytelling-chatbot/src/prompts.py b/examples/storytelling-chatbot/src/prompts.py
index 08abbc93c..068c60e55 100644
--- a/examples/storytelling-chatbot/src/prompts.py
+++ b/examples/storytelling-chatbot/src/prompts.py
@@ -1,35 +1,68 @@
-LLM_INTRO_PROMPT = {
- "role": "system",
- "content": "You are a creative storyteller who loves to tell whimsical, fantastical stories. \
- Your goal is to craft an engaging and fun story. \
- Start by asking the user what kind of story they'd like to hear. Don't provide any examples. \
- Keep your response to only a few sentences.",
-}
-
-
LLM_BASE_PROMPT = {
"role": "system",
- "content": "You are a creative storyteller who loves tell whimsical, fantastical stories. \
- Your goal is to craft an engaging and fun story. \
- Keep all responses short and no more than a few sentences. Include [break] after each sentence of the story. \
- \
- Start each sentence with an image prompt, wrapped in triangle braces, that I can use to generate an illustration representing the upcoming scene. \
- Image prompts should always be wrapped in triangle braces, like this: . \
- You should provide as much descriptive detail in your image prompt as you can to help recreate the current scene depicted by the sentence. \
- For any recurring characters, you should provide a description of them in the image prompt each time, for example: . \
- Please do not include any character names in the image prompts, just their descriptions. \
- Image prompts should focus on key visual attributes of all characters each time, for example . \
- Please use the following structure for your image prompts: characters, setting, action, and mood. \
- Image prompts should be less than 150-200 characters and start in lowercase. \
- \
- Responses should use the format: <...> story sentence [break] <...> story sentence [break] ... \
- After each response, ask me how I'd like the story to continue and wait for my input. \
- Please ensure your responses are less than 3-4 sentences long. \
- Please refrain from using any explicit language or content. Do not tell scary stories.",
+ "content": """You are a creative storyteller who loves tell whimsical, fantastical stories.
+ Your goal is to craft an engaging and fun story.
+ Keep all responses short and no more than a few sentences.
+ Start by asking the user what kind of story they'd like to hear. Don't provide any examples.
+ After they've answered the question, start telling the story. Include three story sentences in your response. Add [break] after each sentence of the story.
+
+ EXAMPLE OUTPUT FORMAT:
+ story sentence 1 [break]
+ story sentence 2 [break]
+ story sentence 3 [break]
+ How would you like the story to continue?
+ END OF EXAMPLE OUTPUT
+
+ Generate three story sentences, then ask what should happen next and wait for my input. You can propose an idea for how the story should proceed, but make sure to tell me I can suggest whatever I want.
+ Please ensure your responses are less than 5 sentences long.
+ Please refrain from using any explicit language or content. Do not tell scary stories.
+ Once you've started telling the story, EVERY RESPONSE should follow the story sentence output format. It is VERY IMPORTANT that you continue to include [break] between story sentences. DO NOT RESPOND without story sentences and break tags.""",
}
-IMAGE_GEN_PROMPT = "illustrative art of %s. In the style of Studio Ghibli. colorful, whimsical, painterly, concept art."
+IMAGE_GEN_PROMPT = "an illustration of %s. colorful, whimsical, painterly, concept art."
CUE_USER_TURN = {"cue": "user_turn"}
CUE_ASSISTANT_TURN = {"cue": "assistant_turn"}
+
+
+""" Start each sentence with an image prompt, wrapped in triangle braces, that I can use to generate an illustration representing the upcoming scene.
+ Image prompts should always be wrapped in triangle braces, like this: .
+ You should provide as much descriptive detail in your image prompt as you can to help recreate the current scene depicted by the sentence.
+ For any recurring characters, you should provide a description of them in the image prompt each time, for example: .
+ Please do not include any character names in the image prompts, just their descriptions.
+ Image prompts should focus on key visual attributes of all characters each time, for example .
+ Please use the following structure for your image prompts: characters, setting, action, and mood.
+ Image prompts should be less than 150-200 characters and start in lowercase."""
+
+FIRST_IMAGE_PROMPT = """You are creating a prompt to generate an image for a child's story book.
+
+You should provide as much descriptive detail in your image prompt as you can to help recreate the current scene depicted by the sentence.
+For any recurring characters, you should provide a description of them in the image prompt each time, for example: .
+Please do not include any character names in the image prompts, just their descriptions.
+Image prompts should focus on key visual attributes of all characters each time, for example .
+Please use the following structure for your image prompts: characters, setting, action, and mood.
+Image prompts should be less than 150-200 characters and start in lowercase.
+
+
+Here's the first page of the story:
+%s
+"""
+
+NEXT_IMAGE_PROMPT = """You are creating a prompt to generate an image for a child's story book.
+
+Here is the text of the story so far:
+%s
+
+Here are the previous image prompts:
+%s
+
+You should provide as much descriptive detail in your image prompt as you can to help recreate the current scene depicted by the sentence.
+For any recurring characters, you should try to use the same description of them in the image prompt each time.
+Please do not include any character names in the image prompts, just their descriptions.
+Image prompts should focus on key visual attributes of all characters each time, for example .
+Please use the following structure for your image prompts: characters, setting, action, and mood.
+Image prompts should be less than 150-200 characters and start in lowercase.
+Here's the next page of the story:
+%s
+"""
diff --git a/examples/studypal/requirements.txt b/examples/studypal/requirements.txt
index 2d6e21042..a2e7b803d 100644
--- a/examples/studypal/requirements.txt
+++ b/examples/studypal/requirements.txt
@@ -1,5 +1,5 @@
beautifulsoup4==4.12.3
pypdf==4.3.1
tiktoken==0.7.0
-pipecat-ai[daily,cartesia,openai,silero]==0.0.40
+pipecat-ai[daily,cartesia,openai,silero]
python-dotenv==1.0.1
diff --git a/examples/studypal/studypal.py b/examples/studypal/studypal.py
index 8e45b609f..0b1496e3d 100644
--- a/examples/studypal/studypal.py
+++ b/examples/studypal/studypal.py
@@ -12,7 +12,6 @@ from pypdf import PdfReader
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer
-from pipecat.frames.frames import EndFrame
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
@@ -113,7 +112,6 @@ async def main():
token,
"studypal",
DailyParams(
- audio_out_sample_rate=44100,
audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True,
@@ -125,7 +123,6 @@ async def main():
api_key=os.getenv("CARTESIA_API_KEY"),
voice_id=os.getenv("CARTESIA_VOICE_ID", "4d2fd738-3b3d-4368-957a-bb4805275bd9"),
# British Narration Lady: 4d2fd738-3b3d-4368-957a-bb4805275bd9
- sample_rate=44100,
)
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o-mini")
@@ -156,7 +153,12 @@ Your task is to help the user understand and learn from this article in 2 senten
]
)
- task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True, enable_metrics=True))
+ task = PipelineTask(
+ pipeline,
+ PipelineParams(
+ audio_out_sample_rate=44100, allow_interruptions=True, enable_metrics=True
+ ),
+ )
@transport.event_handler("on_first_participant_joined")
async def on_first_participant_joined(transport, participant):
@@ -171,7 +173,7 @@ Your task is to help the user understand and learn from this article in 2 senten
@transport.event_handler("on_participant_left")
async def on_participant_left(transport, participant, reason):
- await task.queue_frame(EndFrame())
+ await task.cancel()
runner = PipelineRunner()
diff --git a/examples/telnyx-chatbot/.gitignore b/examples/telnyx-chatbot/.gitignore
new file mode 100644
index 000000000..2bc1403d1
--- /dev/null
+++ b/examples/telnyx-chatbot/.gitignore
@@ -0,0 +1,161 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+# For a library or package, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+# However, in case of collaboration, if having platform-specific dependencies or dependencies
+# having no cross-platform support, pipenv may install dependencies that don't work, or not
+# install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+# This is especially recommended for binary packages to ensure reproducibility, and is more
+# commonly ignored for libraries.
+# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+# in version control.
+# https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+# and can be added to the global gitignore or merged into this file. For a more nuclear
+# option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+runpod.toml
diff --git a/examples/telnyx-chatbot/Dockerfile b/examples/telnyx-chatbot/Dockerfile
new file mode 100644
index 000000000..e9cc69b39
--- /dev/null
+++ b/examples/telnyx-chatbot/Dockerfile
@@ -0,0 +1,20 @@
+# Use an official Python runtime as a parent image
+FROM python:3.10-bullseye
+
+# Set the working directory in the container
+WORKDIR /telnyx-chatbot
+
+# Copy the requirements file into the container
+COPY requirements.txt .
+
+# Install any needed packages specified in requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy the current directory contents into the container
+COPY . .
+
+# Expose the desired port
+EXPOSE 8765
+
+# Run the application
+CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8765"]
diff --git a/examples/telnyx-chatbot/README.md b/examples/telnyx-chatbot/README.md
new file mode 100644
index 000000000..e7400940a
--- /dev/null
+++ b/examples/telnyx-chatbot/README.md
@@ -0,0 +1,112 @@
+# Telnyx Chatbot
+
+This project is a FastAPI-based chatbot that integrates with Telnyx to handle WebSocket connections and provide real-time communication. The project includes endpoints for starting a call and handling WebSocket connections.
+
+## Table of Contents
+
+- [Telnyx Chatbot](#telnyx-chatbot)
+ - [Table of Contents](#table-of-contents)
+ - [Features](#features)
+ - [Requirements](#requirements)
+ - [Installation](#installation)
+ - [Configure Telnyx TeXML application](#configure-telnyx-texml-application)
+ - [Running the Application](#running-the-application)
+ - [Using Python (Option 1)](#using-python-option-1)
+ - [Using Docker (Option 2)](#using-docker-option-2)
+ - [Usage](#usage)
+
+## Features
+
+- **FastAPI**: A modern, fast (high-performance), web framework for building APIs with Python 3.6+.
+- **WebSocket Support**: Real-time communication using WebSockets.
+- **CORS Middleware**: Allowing cross-origin requests for testing.
+- **Dockerized**: Easily deployable using Docker.
+
+## Requirements
+
+- Python 3.10
+- Docker (for containerized deployment)
+- ngrok (for tunneling)
+- Telnyx Account
+
+## Installation
+
+1. **Set up a virtual environment** (optional but recommended):
+
+ ```sh
+ python -m venv venv
+ source venv/bin/activate # On Windows, use `venv\Scripts\activate`
+ ```
+
+2. **Install dependencies**:
+
+ ```sh
+ pip install -r requirements.txt
+ ```
+
+3. **Create .env**:
+ Copy the example environment file and update with your settings:
+
+ ```sh
+ cp env.example .env
+ ```
+
+4. **Install ngrok**:
+ Follow the instructions on the [ngrok website](https://ngrok.com/download) to download and install ngrok.
+
+## Configure Telnyx TeXML application
+
+1. **Start ngrok**:
+ In a new terminal, start ngrok to tunnel the local server:
+
+ ```sh
+ ngrok http 8765
+ ```
+
+2. **Update the Telnyx TeXML applications Webhook**:
+
+ - Go to your TeXML configuration page
+ - Provide the ngrok URL to the Webhook URL field and ensure the POST method is selected
+ - Click Save at the bottom of the page
+
+3. **Configure streams.xml**:
+ - Copy the template file to create your local version:
+ ```sh
+ cp templates/streams.xml.template templates/streams.xml
+ ```
+ - In `templates/streams.xml`, replace `` with your ngrok URL (without `https://`)
+ - The final URL should look like: `wss://abc123.ngrok.io/ws`
+ - The encoding (`bidirectionalCodec`) should be `PCMU` or `PCMA` depending on your needs. Based on selected encoding, set the outbound_encoding in `server.py` when the bot is initialized.
+ - The inbound encoding can be controlled from the application configuration for inbound calls and dial/transfer commands for outbound calls.
+
+## Running the Application
+
+Choose one of these two methods to run the application:
+
+### Using Python (Option 1)
+
+**Run the FastAPI application**:
+
+```sh
+# Make sure you’re in the project directory and your virtual environment is activated
+python server.py
+```
+
+### Using Docker (Option 2)
+
+1. **Build the Docker image**:
+
+ ```sh
+ docker build -t telnyx-chatbot .
+ ```
+
+2. **Run the Docker container**:
+ ```sh
+ docker run -it --rm -p 8765:8765 telnyx-chatbot
+ ```
+
+The server will start on port 8765. Keep this running while you test with Telnyx.
+
+## Usage
+
+To start a call, simply make a call to your configured Telnyx phone number. The webhook URL will direct the call to your FastAPI application, which will handle it accordingly.
diff --git a/examples/telnyx-chatbot/bot.py b/examples/telnyx-chatbot/bot.py
new file mode 100644
index 000000000..8243e05a4
--- /dev/null
+++ b/examples/telnyx-chatbot/bot.py
@@ -0,0 +1,103 @@
+#
+# Copyright (c) 2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import os
+import sys
+
+from dotenv import load_dotenv
+from loguru import logger
+
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.serializers.telnyx import TelnyxFrameSerializer
+from pipecat.services.cartesia import CartesiaTTSService
+from pipecat.services.deepgram import DeepgramSTTService
+from pipecat.services.openai import OpenAILLMService
+from pipecat.transports.network.fastapi_websocket import (
+ FastAPIWebsocketParams,
+ FastAPIWebsocketTransport,
+)
+
+load_dotenv(override=True)
+
+logger.remove(0)
+logger.add(sys.stderr, level="DEBUG")
+
+
+async def run_bot(
+ websocket_client,
+ stream_id: str,
+ outbound_encoding: str,
+ inbound_encoding: str,
+):
+ transport = FastAPIWebsocketTransport(
+ websocket=websocket_client,
+ params=FastAPIWebsocketParams(
+ audio_out_enabled=True,
+ add_wav_header=False,
+ vad_enabled=True,
+ vad_analyzer=SileroVADAnalyzer(),
+ vad_audio_passthrough=True,
+ serializer=TelnyxFrameSerializer(stream_id, outbound_encoding, inbound_encoding),
+ ),
+ )
+
+ llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
+
+ stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
+
+ tts = CartesiaTTSService(
+ api_key=os.getenv("CARTESIA_API_KEY"),
+ voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady
+ )
+
+ messages = [
+ {
+ "role": "system",
+ "content": "You are a helpful LLM in an audio call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
+ },
+ ]
+
+ context = OpenAILLMContext(messages)
+ context_aggregator = llm.create_context_aggregator(context)
+
+ pipeline = Pipeline(
+ [
+ transport.input(), # Websocket input from client
+ stt, # Speech-To-Text
+ context_aggregator.user(),
+ llm, # LLM
+ tts, # Text-To-Speech
+ transport.output(), # Websocket output to client
+ context_aggregator.assistant(),
+ ]
+ )
+
+ task = PipelineTask(
+ pipeline,
+ params=PipelineParams(
+ audio_in_sample_rate=8000,
+ audio_out_sample_rate=8000,
+ allow_interruptions=True,
+ ),
+ )
+
+ @transport.event_handler("on_client_connected")
+ async def on_client_connected(transport, client):
+ # Kick off the conversation.
+ messages.append({"role": "system", "content": "Please introduce yourself to the user."})
+ await task.queue_frames([context_aggregator.user().get_context_frame()])
+
+ @transport.event_handler("on_client_disconnected")
+ async def on_client_disconnected(transport, client):
+ await task.cancel()
+
+ runner = PipelineRunner(handle_sigint=False)
+
+ await runner.run(task)
diff --git a/examples/telnyx-chatbot/env.example b/examples/telnyx-chatbot/env.example
new file mode 100644
index 000000000..5de3bace2
--- /dev/null
+++ b/examples/telnyx-chatbot/env.example
@@ -0,0 +1,3 @@
+OPENAI_API_KEY=
+DEEPGRAM_API_KEY=
+ELEVENLABS_API_KEY=
diff --git a/examples/telnyx-chatbot/requirements.txt b/examples/telnyx-chatbot/requirements.txt
new file mode 100644
index 000000000..630a98dd6
--- /dev/null
+++ b/examples/telnyx-chatbot/requirements.txt
@@ -0,0 +1,5 @@
+pipecat-ai[openai,silero,deepgram,elevenlabs]
+fastapi
+uvicorn
+python-dotenv
+loguru
diff --git a/examples/telnyx-chatbot/server.py b/examples/telnyx-chatbot/server.py
new file mode 100644
index 000000000..7bc1c0615
--- /dev/null
+++ b/examples/telnyx-chatbot/server.py
@@ -0,0 +1,46 @@
+#
+# Copyright (c) 2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import json
+
+import uvicorn
+from bot import run_bot
+from fastapi import FastAPI, WebSocket
+from fastapi.middleware.cors import CORSMiddleware
+from starlette.responses import HTMLResponse
+
+app = FastAPI()
+
+app.add_middleware(
+ CORSMiddleware,
+ allow_origins=["*"], # Allow all origins for testing
+ allow_credentials=True,
+ allow_methods=["*"],
+ allow_headers=["*"],
+)
+
+
+@app.post("/")
+async def start_call():
+ print("POST TeXML")
+ return HTMLResponse(content=open("templates/streams.xml").read(), media_type="application/xml")
+
+
+@app.websocket("/ws")
+async def websocket_endpoint(websocket: WebSocket):
+ await websocket.accept()
+ start_data = websocket.iter_text()
+ await start_data.__anext__()
+ call_data = json.loads(await start_data.__anext__())
+ print(call_data, flush=True)
+ stream_id = call_data["stream_id"]
+ outbound_encoding = call_data["start"]["media_format"]["encoding"]
+ print("WebSocket connection accepted")
+ await run_bot(websocket, stream_id, outbound_encoding, "PCMU")
+
+
+if __name__ == "__main__":
+ uvicorn.run(app, host="0.0.0.0", port=8765)
diff --git a/examples/telnyx-chatbot/templates/streams.xml.template b/examples/telnyx-chatbot/templates/streams.xml.template
new file mode 100644
index 000000000..89826ff33
--- /dev/null
+++ b/examples/telnyx-chatbot/templates/streams.xml.template
@@ -0,0 +1,7 @@
+
+
+
+
+
diff --git a/examples/translation-chatbot/bot.py b/examples/translation-chatbot/bot.py
index cf3d891b2..d35cf7af2 100644
--- a/examples/translation-chatbot/bot.py
+++ b/examples/translation-chatbot/bot.py
@@ -16,7 +16,6 @@ from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import (
- EndFrame,
Frame,
LLMMessagesFrame,
TranscriptionFrame,
@@ -26,7 +25,6 @@ from pipecat.frames.frames import (
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask
-from pipecat.processors.aggregators.llm_response import LLMFullResponseAggregator
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.processors.transcript_processor import TranscriptProcessor
@@ -190,7 +188,7 @@ async def main():
@transport.event_handler("on_participant_left")
async def on_participant_left(transport, participant, reason):
- await task.queue_frame(EndFrame())
+ await task.cancel()
runner = PipelineRunner()
diff --git a/examples/twilio-chatbot/README.md b/examples/twilio-chatbot/README.md
index 32bd4de49..d06fd7f85 100644
--- a/examples/twilio-chatbot/README.md
+++ b/examples/twilio-chatbot/README.md
@@ -107,3 +107,34 @@ The server will start on port 8765. Keep this running while you test with Twilio
## Usage
To start a call, simply make a call to your configured Twilio phone number. The webhook URL will direct the call to your FastAPI application, which will handle it accordingly.
+
+## Testing
+
+It is also possible to automatically test the server without making phone calls by using a software client.
+
+First, update `templates/streams.xml` to point to your server's websocket endpoint. For example:
+
+```
+
+
+
+
+
+```
+
+Then, start the server with `-t` to indicate we are testing:
+
+```sh
+# Make sure you’re in the project directory and your virtual environment is activated
+python server.py -t
+```
+
+Finally, just point the client to the server's URL:
+
+```sh
+python client.py -u http://localhost:8765 -c 2
+```
+
+where `-c` allows you to create multiple concurrent clients.
diff --git a/examples/twilio-chatbot/bot.py b/examples/twilio-chatbot/bot.py
index aed7041f8..6d5953954 100644
--- a/examples/twilio-chatbot/bot.py
+++ b/examples/twilio-chatbot/bot.py
@@ -4,18 +4,23 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
+import datetime
+import io
import os
import sys
+import wave
+import aiofiles
from dotenv import load_dotenv
+from fastapi import WebSocket
from loguru import logger
from pipecat.audio.vad.silero import SileroVADAnalyzer
-from pipecat.frames.frames import EndFrame
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor
from pipecat.serializers.twilio import TwilioFrameSerializer
from pipecat.services.cartesia import CartesiaTTSService
from pipecat.services.deepgram import DeepgramSTTService
@@ -31,10 +36,29 @@ logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
-async def run_bot(websocket_client, stream_sid):
+async def save_audio(server_name: str, audio: bytes, sample_rate: int, num_channels: int):
+ if len(audio) > 0:
+ filename = (
+ f"{server_name}_recording_{datetime.datetime.now().strftime('%Y%m%d_%H%M%S')}.wav"
+ )
+ with io.BytesIO() as buffer:
+ with wave.open(buffer, "wb") as wf:
+ wf.setsampwidth(2)
+ wf.setnchannels(num_channels)
+ wf.setframerate(sample_rate)
+ wf.writeframes(audio)
+ async with aiofiles.open(filename, "wb") as file:
+ await file.write(buffer.getvalue())
+ logger.info(f"Merged audio saved to {filename}")
+ else:
+ logger.info("No audio data to save")
+
+
+async def run_bot(websocket_client: WebSocket, stream_sid: str, testing: bool):
transport = FastAPIWebsocketTransport(
websocket=websocket_client,
params=FastAPIWebsocketParams(
+ audio_in_enabled=True,
audio_out_enabled=True,
add_wav_header=False,
vad_enabled=True,
@@ -46,23 +70,28 @@ async def run_bot(websocket_client, stream_sid):
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
- stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
+ stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"), audio_passthrough=True)
tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady
+ push_silence_after_stop=testing,
)
messages = [
{
"role": "system",
- "content": "You are a helpful LLM in an audio call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
+ "content": "You are an elementary teacher in an audio call. Your output will be converted to audio so don't include special characters in your answers. Respond to what the student said in a short short sentence.",
},
]
context = OpenAILLMContext(messages)
context_aggregator = llm.create_context_aggregator(context)
+ # NOTE: Watch out! This will save all the conversation in memory. You can
+ # pass `buffer_size` to get periodic callbacks.
+ audiobuffer = AudioBufferProcessor()
+
pipeline = Pipeline(
[
transport.input(), # Websocket input from client
@@ -71,22 +100,39 @@ async def run_bot(websocket_client, stream_sid):
llm, # LLM
tts, # Text-To-Speech
transport.output(), # Websocket output to client
+ audiobuffer, # Used to buffer the audio in the pipeline
context_aggregator.assistant(),
]
)
- task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True))
+ task = PipelineTask(
+ pipeline,
+ params=PipelineParams(
+ audio_in_sample_rate=8000, audio_out_sample_rate=8000, allow_interruptions=True
+ ),
+ )
@transport.event_handler("on_client_connected")
async def on_client_connected(transport, client):
+ # Start recording.
+ await audiobuffer.start_recording()
# Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
- await task.queue_frames([EndFrame()])
+ await task.cancel()
- runner = PipelineRunner(handle_sigint=False)
+ @audiobuffer.event_handler("on_audio_data")
+ async def on_audio_data(buffer, audio, sample_rate, num_channels):
+ server_name = f"server_{websocket_client.client.port}"
+ await save_audio(server_name, audio, sample_rate, num_channels)
+
+ # We use `handle_sigint=False` because `uvicorn` is controlling keyboard
+ # interruptions. We use `force_gc=True` to force garbage collection after
+ # the runner finishes running a task which could be useful for long running
+ # applications with multiple clients connecting.
+ runner = PipelineRunner(handle_sigint=False, force_gc=True)
await runner.run(task)
diff --git a/examples/twilio-chatbot/client.py b/examples/twilio-chatbot/client.py
new file mode 100644
index 000000000..7cd90e2c8
--- /dev/null
+++ b/examples/twilio-chatbot/client.py
@@ -0,0 +1,199 @@
+#
+# Copyright (c) 2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import argparse
+import asyncio
+import datetime
+import io
+import os
+import sys
+import wave
+import xml.etree.ElementTree as ET
+from uuid import uuid4
+
+import aiofiles
+import aiohttp
+from dotenv import load_dotenv
+from loguru import logger
+
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.audio.vad.vad_analyzer import VADParams
+from pipecat.frames.frames import EndFrame, TransportMessageUrgentFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor
+from pipecat.serializers.twilio import TwilioFrameSerializer
+from pipecat.services.cartesia import CartesiaTTSService
+from pipecat.services.deepgram import DeepgramSTTService
+from pipecat.services.openai import OpenAILLMService
+from pipecat.transports.network.websocket_client import (
+ WebsocketClientParams,
+ WebsocketClientTransport,
+)
+
+load_dotenv(override=True)
+
+logger.remove(0)
+logger.add(sys.stderr, level="DEBUG")
+
+
+DEFAULT_CLIENT_DURATION = 30
+
+
+async def download_twiml(server_url: str) -> str:
+ # TODO(aleix): add error checking.
+ async with aiohttp.ClientSession() as session:
+ async with session.post(server_url) as response:
+ return await response.text()
+
+
+def get_stream_url_from_twiml(twiml: str) -> str:
+ root = ET.fromstring(twiml)
+ # TODO(aleix): add error checking.
+ stream_element = root.find(".//Stream") # Finds the first element
+ url = stream_element.get("url")
+ return url
+
+
+async def save_audio(client_name: str, audio: bytes, sample_rate: int, num_channels: int):
+ if len(audio) > 0:
+ filename = (
+ f"{client_name}_recording_{datetime.datetime.now().strftime('%Y%m%d_%H%M%S')}.wav"
+ )
+ with io.BytesIO() as buffer:
+ with wave.open(buffer, "wb") as wf:
+ wf.setsampwidth(2)
+ wf.setnchannels(num_channels)
+ wf.setframerate(sample_rate)
+ wf.writeframes(audio)
+ async with aiofiles.open(filename, "wb") as file:
+ await file.write(buffer.getvalue())
+ logger.info(f"Merged audio saved to {filename}")
+ else:
+ logger.info("No audio data to save")
+
+
+async def run_client(client_name: str, server_url: str, duration_secs: int):
+ twiml = await download_twiml(server_url)
+
+ stream_url = get_stream_url_from_twiml(twiml)
+
+ stream_sid = str(uuid4())
+
+ transport = WebsocketClientTransport(
+ uri=stream_url,
+ params=WebsocketClientParams(
+ audio_in_enabled=True,
+ audio_out_enabled=True,
+ add_wav_header=False,
+ serializer=TwilioFrameSerializer(stream_sid),
+ vad_enabled=True,
+ vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=1.5)),
+ vad_audio_passthrough=True,
+ ),
+ )
+
+ llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
+
+ # We let the audio passthrough so we can record the conversation.
+ stt = DeepgramSTTService(
+ api_key=os.getenv("DEEPGRAM_API_KEY"),
+ audio_passthrough=True,
+ )
+
+ tts = CartesiaTTSService(
+ api_key=os.getenv("CARTESIA_API_KEY"),
+ voice_id="e13cae5c-ec59-4f71-b0a6-266df3c9bb8e", # Madame Mischief
+ push_silence_after_stop=True,
+ )
+
+ messages = [
+ {
+ "role": "system",
+ "content": "You are an 8 year old child. A teacher will explain you new concepts you want to know about. Feel free to change topics whnever you want. Once you are taught something you need to keep asking for clarifications if you think someone your age would not understand what you are being taught.",
+ },
+ ]
+
+ context = OpenAILLMContext(messages)
+ context_aggregator = llm.create_context_aggregator(context)
+
+ # NOTE: Watch out! This will save all the conversation in memory. You can
+ # pass `buffer_size` to get periodic callbacks.
+ audiobuffer = AudioBufferProcessor()
+
+ pipeline = Pipeline(
+ [
+ transport.input(), # Websocket input from server
+ stt, # Speech-To-Text
+ context_aggregator.user(),
+ llm, # LLM
+ tts, # Text-To-Speech
+ transport.output(), # Websocket output to server
+ audiobuffer, # Used to buffer the audio in the pipeline
+ context_aggregator.assistant(),
+ ]
+ )
+
+ task = PipelineTask(
+ pipeline,
+ params=PipelineParams(
+ audio_in_sample_rate=8000, audio_out_sample_rate=8000, allow_interruptions=True
+ ),
+ )
+
+ @transport.event_handler("on_connected")
+ async def on_connected(transport: WebsocketClientTransport, client):
+ # Start recording.
+ await audiobuffer.start_recording()
+
+ message = TransportMessageUrgentFrame(
+ message={"event": "connected", "protocol": "Call", "version": "1.0.0"}
+ )
+ await transport.output().send_message(message)
+
+ message = TransportMessageUrgentFrame(
+ message={"event": "start", "streamSid": stream_sid, "start": {"streamSid": stream_sid}}
+ )
+ await transport.output().send_message(message)
+
+ @audiobuffer.event_handler("on_audio_data")
+ async def on_audio_data(buffer, audio, sample_rate, num_channels):
+ await save_audio(client_name, audio, sample_rate, num_channels)
+
+ async def end_call():
+ await asyncio.sleep(duration_secs)
+ await task.queue_frame(EndFrame())
+
+ runner = PipelineRunner()
+
+ await asyncio.gather(runner.run(task), end_call())
+
+
+async def main():
+ parser = argparse.ArgumentParser(description="Pipecat Twilio Chatbot Client")
+ parser.add_argument("-u", "--url", type=str, required=True, help="specify the server URL")
+ parser.add_argument(
+ "-c", "--clients", type=int, required=True, help="number of concurrent clients"
+ )
+ parser.add_argument(
+ "-d",
+ "--duration",
+ type=int,
+ default=DEFAULT_CLIENT_DURATION,
+ help=f"duration of each client in seconds (default: {DEFAULT_CLIENT_DURATION})",
+ )
+ args, _ = parser.parse_known_args()
+
+ clients = []
+ for i in range(args.clients):
+ clients.append(asyncio.create_task(run_client(f"client_{i}", args.url, args.duration)))
+ await asyncio.gather(*clients)
+
+
+if __name__ == "__main__":
+ asyncio.run(main())
diff --git a/examples/twilio-chatbot/server.py b/examples/twilio-chatbot/server.py
index c59b6e2ee..d65015b5c 100644
--- a/examples/twilio-chatbot/server.py
+++ b/examples/twilio-chatbot/server.py
@@ -1,3 +1,10 @@
+#
+# Copyright (c) 2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import argparse
import json
import uvicorn
@@ -32,8 +39,16 @@ async def websocket_endpoint(websocket: WebSocket):
print(call_data, flush=True)
stream_sid = call_data["start"]["streamSid"]
print("WebSocket connection accepted")
- await run_bot(websocket, stream_sid)
+ await run_bot(websocket, stream_sid, app.state.testing)
if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description="Pipecat Twilio Chatbot Server")
+ parser.add_argument(
+ "-t", "--test", action="store_true", default=False, help="set the server in testing mode"
+ )
+ args, _ = parser.parse_known_args()
+
+ app.state.testing = args.test
+
uvicorn.run(app, host="0.0.0.0", port=8765)
diff --git a/examples/websocket-server/bot.py b/examples/websocket-server/bot.py
index a36339fff..9a0de15c6 100644
--- a/examples/websocket-server/bot.py
+++ b/examples/websocket-server/bot.py
@@ -17,6 +17,7 @@ from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.serializers.protobuf import ProtobufFrameSerializer
from pipecat.services.cartesia import CartesiaTTSService
from pipecat.services.deepgram import DeepgramSTTService
from pipecat.services.openai import OpenAILLMService
@@ -80,7 +81,7 @@ class SessionTimeoutHandler:
async def main():
transport = WebsocketServerTransport(
params=WebsocketServerParams(
- audio_out_sample_rate=16000,
+ serializer=ProtobufFrameSerializer(),
audio_out_enabled=True,
add_wav_header=True,
vad_enabled=True,
@@ -97,7 +98,6 @@ async def main():
tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady
- sample_rate=16000,
)
messages = [
@@ -122,7 +122,12 @@ async def main():
]
)
- task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True))
+ task = PipelineTask(
+ pipeline,
+ params=PipelineParams(
+ audio_in_sample_rate=16000, audio_out_sample_rate=16000, allow_interruptions=True
+ ),
+ )
@transport.event_handler("on_client_connected")
async def on_client_connected(transport, client):
diff --git a/pyproject.toml b/pyproject.toml
index 38bf6d902..31efcae2a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -32,7 +32,8 @@ dependencies = [
"protobuf~=5.29.3",
"pydantic~=2.10.5",
"pyloudnorm~=0.1.1",
- "resampy~=0.4.3"
+ "resampy~=0.4.3",
+ "soxr~=0.5.0"
]
[project.urls]
@@ -40,7 +41,7 @@ Source = "https://github.com/pipecat-ai/pipecat"
Website = "https://pipecat.ai"
[project.optional-dependencies]
-anthropic = [ "anthropic~=0.39.0" ]
+anthropic = [ "anthropic~=0.45.2" ]
assemblyai = [ "assemblyai~=0.36.0" ]
aws = [ "boto3~=1.35.99" ]
azure = [ "azure-cognitiveservices-speech~=1.42.0", "openai~=1.59.6" ]
@@ -54,7 +55,7 @@ elevenlabs = [ "websockets~=13.1" ]
fal = [ "fal-client~=0.5.6" ]
fish = [ "ormsgpack~=1.7.0", "websockets~=13.1" ]
gladia = [ "websockets~=13.1" ]
-google = [ "google-generativeai~=0.8.3", "google-cloud-texttospeech~=2.24.0" ]
+google = [ "google-generativeai~=0.8.3", "google-cloud-texttospeech~=2.24.0", "google-genai~=0.7.0" ]
grok = [ "openai~=1.59.6" ]
groq = [ "openai~=1.59.6" ]
gstreamer = [ "pygobject~=3.50.0" ]
@@ -69,7 +70,7 @@ moondream = [ "einops~=0.8.0", "timm~=1.0.13", "transformers~=4.48.0" ]
nim = [ "openai~=1.59.6" ]
noisereduce = [ "noisereduce~=3.0.3" ]
openai = [ "openai~=1.59.6", "websockets~=13.1", "python-deepcompare~=2.1.0" ]
-openpipe = [ "openpipe~=4.43.0" ]
+openpipe = [ "openpipe~=4.45.0" ]
playht = [ "pyht~=0.1.6", "websockets~=13.1" ]
riva = [ "nvidia-riva-client~=2.18.0" ]
silero = [ "onnxruntime~=1.20.1" ]
@@ -85,7 +86,13 @@ openrouter = [ "openai~=1.59.6" ]
where = ["src"]
[tool.pytest.ini_options]
+addopts = "--verbose"
+testpaths = ["tests"]
pythonpath = ["src"]
+asyncio_default_fixture_loop_scope = "function"
+filterwarnings = [
+ "ignore:'audioop' is deprecated:DeprecationWarning",
+]
[tool.setuptools_scm]
local_scheme = "no-local-version"
diff --git a/src/pipecat/audio/filters/krisp_filter.py b/src/pipecat/audio/filters/krisp_filter.py
index 194dd848d..b23a46b65 100644
--- a/src/pipecat/audio/filters/krisp_filter.py
+++ b/src/pipecat/audio/filters/krisp_filter.py
@@ -20,6 +20,22 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
+class KrispProcessorManager:
+ """
+ Ensures that only one KrispAudioProcessor instance exists for the entire program.
+ """
+
+ _krisp_instance = None
+
+ @classmethod
+ def get_processor(cls, sample_rate: int, sample_type: str, channels: int, model_path: str):
+ if cls._krisp_instance is None:
+ cls._krisp_instance = KrispAudioProcessor(
+ sample_rate, sample_type, channels, model_path
+ )
+ return cls._krisp_instance
+
+
class KrispFilter(BaseAudioFilter):
def __init__(
self, sample_type: str = "PCM_16", channels: int = 1, model_path: str = None
@@ -48,7 +64,7 @@ class KrispFilter(BaseAudioFilter):
async def start(self, sample_rate: int):
self._sample_rate = sample_rate
- self._krisp_processor = KrispAudioProcessor(
+ self._krisp_processor = KrispProcessorManager.get_processor(
self._sample_rate, self._sample_type, self._channels, self._model_path
)
diff --git a/src/pipecat/audio/resamplers/__init__.py b/src/pipecat/audio/resamplers/__init__.py
new file mode 100644
index 000000000..8b1378917
--- /dev/null
+++ b/src/pipecat/audio/resamplers/__init__.py
@@ -0,0 +1 @@
+
diff --git a/src/pipecat/audio/resamplers/base_audio_resampler.py b/src/pipecat/audio/resamplers/base_audio_resampler.py
new file mode 100644
index 000000000..6afbbcbfe
--- /dev/null
+++ b/src/pipecat/audio/resamplers/base_audio_resampler.py
@@ -0,0 +1,30 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+from abc import ABC, abstractmethod
+
+
+class BaseAudioResampler(ABC):
+ """Abstract base class for audio resampling. This class defines an
+ interface for audio resampling implementations.
+ """
+
+ @abstractmethod
+ async def resample(self, audio: bytes, in_rate: int, out_rate: int) -> bytes:
+ """
+ Resamples the given audio data to a different sample rate.
+
+ This is an abstract method that must be implemented in subclasses.
+
+ Parameters:
+ audio (bytes): The audio data to be resampled, represented as a byte string.
+ in_rate (int): The original sample rate of the audio data (in Hz).
+ out_rate (int): The desired sample rate for the resampled audio data (in Hz).
+
+ Returns:
+ bytes: The resampled audio data as a byte string.
+ """
+ pass
diff --git a/src/pipecat/audio/resamplers/resampy_resampler.py b/src/pipecat/audio/resamplers/resampy_resampler.py
new file mode 100644
index 000000000..8c053fc3b
--- /dev/null
+++ b/src/pipecat/audio/resamplers/resampy_resampler.py
@@ -0,0 +1,25 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import numpy as np
+import resampy
+
+from pipecat.audio.resamplers.base_audio_resampler import BaseAudioResampler
+
+
+class ResampyResampler(BaseAudioResampler):
+ """Audio resampler implementation using the resampy library."""
+
+ def __init__(self, **kwargs):
+ pass
+
+ async def resample(self, audio: bytes, in_rate: int, out_rate: int) -> bytes:
+ if in_rate == out_rate:
+ return audio
+ audio_data = np.frombuffer(audio, dtype=np.int16)
+ resampled_audio = resampy.resample(audio_data, in_rate, out_rate, filter="kaiser_fast")
+ result = resampled_audio.astype(np.int16).tobytes()
+ return result
diff --git a/src/pipecat/audio/resamplers/soxr_resampler.py b/src/pipecat/audio/resamplers/soxr_resampler.py
new file mode 100644
index 000000000..88edb84eb
--- /dev/null
+++ b/src/pipecat/audio/resamplers/soxr_resampler.py
@@ -0,0 +1,25 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import numpy as np
+import soxr
+
+from pipecat.audio.resamplers.base_audio_resampler import BaseAudioResampler
+
+
+class SOXRAudioResampler(BaseAudioResampler):
+ """Audio resampler implementation using the SoX resampler library."""
+
+ def __init__(self, **kwargs):
+ pass
+
+ async def resample(self, audio: bytes, in_rate: int, out_rate: int) -> bytes:
+ if in_rate == out_rate:
+ return audio
+ audio_data = np.frombuffer(audio, dtype=np.int16)
+ resampled_audio = soxr.resample(audio_data, in_rate, out_rate, quality="VHQ")
+ result = resampled_audio.astype(np.int16).tobytes()
+ return result
diff --git a/src/pipecat/audio/utils.py b/src/pipecat/audio/utils.py
index 143f05c28..7c4c25fcc 100644
--- a/src/pipecat/audio/utils.py
+++ b/src/pipecat/audio/utils.py
@@ -8,14 +8,30 @@ import audioop
import numpy as np
import pyloudnorm as pyln
-import resampy
+import soxr
+
+from pipecat.audio.resamplers.base_audio_resampler import BaseAudioResampler
+from pipecat.audio.resamplers.soxr_resampler import SOXRAudioResampler
+
+
+def create_default_resampler(**kwargs) -> BaseAudioResampler:
+ return SOXRAudioResampler(**kwargs)
def resample_audio(audio: bytes, original_rate: int, target_rate: int) -> bytes:
+ import warnings
+
+ with warnings.catch_warnings():
+ warnings.simplefilter("always")
+ warnings.warn(
+ "'resample_audio()' is deprecated, use 'create_default_resampler()' instead.",
+ DeprecationWarning,
+ )
+
if original_rate == target_rate:
return audio
audio_data = np.frombuffer(audio, dtype=np.int16)
- resampled_audio = resampy.resample(audio_data, original_rate, target_rate)
+ resampled_audio = soxr.resample(audio_data, original_rate, target_rate)
return resampled_audio.astype(np.int16).tobytes()
@@ -75,21 +91,45 @@ def exp_smoothing(value: float, prev_value: float, factor: float) -> float:
return prev_value + factor * (value - prev_value)
-def ulaw_to_pcm(ulaw_bytes: bytes, in_sample_rate: int, out_sample_rate: int):
+async def ulaw_to_pcm(
+ ulaw_bytes: bytes, in_rate: int, out_rate: int, resampler: BaseAudioResampler
+):
# Convert μ-law to PCM
in_pcm_bytes = audioop.ulaw2lin(ulaw_bytes, 2)
# Resample
- out_pcm_bytes = resample_audio(in_pcm_bytes, in_sample_rate, out_sample_rate)
+ out_pcm_bytes = await resampler.resample(in_pcm_bytes, in_rate, out_rate)
return out_pcm_bytes
-def pcm_to_ulaw(pcm_bytes: bytes, in_sample_rate: int, out_sample_rate: int):
+async def pcm_to_ulaw(pcm_bytes: bytes, in_rate: int, out_rate: int, resampler: BaseAudioResampler):
# Resample
- in_pcm_bytes = resample_audio(pcm_bytes, in_sample_rate, out_sample_rate)
+ in_pcm_bytes = await resampler.resample(pcm_bytes, in_rate, out_rate)
# Convert PCM to μ-law
- ulaw_bytes = audioop.lin2ulaw(in_pcm_bytes, 2)
+ out_ulaw_bytes = audioop.lin2ulaw(in_pcm_bytes, 2)
- return ulaw_bytes
+ return out_ulaw_bytes
+
+
+async def alaw_to_pcm(
+ alaw_bytes: bytes, in_rate: int, out_rate: int, resampler: BaseAudioResampler
+) -> bytes:
+ # Convert a-law to PCM
+ in_pcm_bytes = audioop.alaw2lin(alaw_bytes, 2)
+
+ # Resample
+ out_pcm_bytes = await resampler.resample(in_pcm_bytes, in_rate, out_rate)
+
+ return out_pcm_bytes
+
+
+async def pcm_to_alaw(pcm_bytes: bytes, in_rate: int, out_rate: int, resampler: BaseAudioResampler):
+ # Resample
+ in_pcm_bytes = await resampler.resample(pcm_bytes, in_rate, out_rate)
+
+ # Convert PCM to μ-law
+ out_alaw_bytes = audioop.lin2alaw(in_pcm_bytes, 2)
+
+ return out_alaw_bytes
diff --git a/src/pipecat/audio/vad/silero.py b/src/pipecat/audio/vad/silero.py
index fab01ae70..a5db2a0ae 100644
--- a/src/pipecat/audio/vad/silero.py
+++ b/src/pipecat/audio/vad/silero.py
@@ -5,6 +5,7 @@
#
import time
+from typing import Optional
import numpy as np
from loguru import logger
@@ -104,11 +105,8 @@ class SileroOnnxModel:
class SileroVADAnalyzer(VADAnalyzer):
- def __init__(self, *, sample_rate: int = 16000, params: VADParams = VADParams()):
- super().__init__(sample_rate=sample_rate, num_channels=1, params=params)
-
- if sample_rate != 16000 and sample_rate != 8000:
- raise ValueError("Silero VAD sample rate needs to be 16000 or 8000")
+ def __init__(self, *, sample_rate: Optional[int] = None, params: VADParams = VADParams()):
+ super().__init__(sample_rate=sample_rate, params=params)
logger.debug("Loading Silero VAD model...")
@@ -138,6 +136,12 @@ class SileroVADAnalyzer(VADAnalyzer):
# VADAnalyzer
#
+ def set_sample_rate(self, sample_rate: int):
+ if sample_rate != 16000 and sample_rate != 8000:
+ raise ValueError("Silero VAD sample rate needs to be 16000 or 8000")
+
+ super().set_sample_rate(sample_rate)
+
def num_frames_required(self) -> int:
return 512 if self.sample_rate == 16000 else 256
@@ -159,5 +163,5 @@ class SileroVADAnalyzer(VADAnalyzer):
return new_confidence
except Exception as e:
# This comes from an empty audio array
- logger.exception(f"Error analyzing audio with Silero VAD: {e}")
+ logger.error(f"Error analyzing audio with Silero VAD: {e}")
return 0
diff --git a/src/pipecat/audio/vad/vad_analyzer.py b/src/pipecat/audio/vad/vad_analyzer.py
index a6c4f70a1..b5d9b0ba2 100644
--- a/src/pipecat/audio/vad/vad_analyzer.py
+++ b/src/pipecat/audio/vad/vad_analyzer.py
@@ -6,6 +6,7 @@
from abc import abstractmethod
from enum import Enum
+from typing import Optional
from loguru import logger
from pydantic import BaseModel
@@ -33,11 +34,11 @@ class VADParams(BaseModel):
class VADAnalyzer:
- def __init__(self, *, sample_rate: int, num_channels: int, params: VADParams):
- self._sample_rate = sample_rate
- self._num_channels = num_channels
-
- self.set_params(params)
+ def __init__(self, *, sample_rate: Optional[int] = None, params: VADParams):
+ self._init_sample_rate = sample_rate
+ self._sample_rate = 0
+ self._params = params
+ self._num_channels = 1
self._vad_buffer = b""
@@ -65,13 +66,17 @@ class VADAnalyzer:
def voice_confidence(self, buffer) -> float:
pass
+ def set_sample_rate(self, sample_rate: int):
+ self._sample_rate = self._init_sample_rate or sample_rate
+ self.set_params(self._params)
+
def set_params(self, params: VADParams):
logger.info(f"Setting VAD params to: {params}")
self._params = params
self._vad_frames = self.num_frames_required()
self._vad_frames_num_bytes = self._vad_frames * self._num_channels * 2
- vad_frames_per_sec = self._vad_frames / self._sample_rate
+ vad_frames_per_sec = self._vad_frames / self.sample_rate
self._vad_start_frames = round(self._params.start_secs / vad_frames_per_sec)
self._vad_stop_frames = round(self._params.stop_secs / vad_frames_per_sec)
@@ -80,7 +85,7 @@ class VADAnalyzer:
self._vad_state: VADState = VADState.QUIET
def _get_smoothed_volume(self, audio: bytes) -> float:
- volume = calculate_audio_volume(audio, self._sample_rate)
+ volume = calculate_audio_volume(audio, self.sample_rate)
return exp_smoothing(volume, self._prev_volume, self._smoothing_factor)
def analyze_audio(self, buffer) -> VADState:
diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py
index 695cc4c74..30549f739 100644
--- a/src/pipecat/frames/frames.py
+++ b/src/pipecat/frames/frames.py
@@ -6,12 +6,24 @@
from dataclasses import dataclass, field
from enum import Enum
-from typing import TYPE_CHECKING, Any, Awaitable, Callable, List, Literal, Mapping, Optional, Tuple
+from typing import (
+ TYPE_CHECKING,
+ Any,
+ Awaitable,
+ Callable,
+ Dict,
+ List,
+ Literal,
+ Mapping,
+ Optional,
+ Tuple,
+)
from pipecat.audio.vad.vad_analyzer import VADParams
from pipecat.clocks.base_clock import BaseClock
from pipecat.metrics.metrics import MetricsData
from pipecat.transcriptions.language import Language
+from pipecat.utils.asyncio import TaskManager
from pipecat.utils.time import nanoseconds_to_str
from pipecat.utils.utils import obj_count, obj_id
@@ -47,11 +59,13 @@ class Frame:
id: int = field(init=False)
name: str = field(init=False)
pts: Optional[int] = field(init=False)
+ metadata: Dict[str, Any] = field(init=False)
def __post_init__(self):
self.id: int = obj_id()
self.name: str = f"{self.__class__.__name__}#{obj_count(self)}"
self.pts: Optional[int] = None
+ self.metadata: Dict[str, Any] = {}
def __str__(self):
return self.name
@@ -394,12 +408,26 @@ class TransportMessageFrame(DataFrame):
@dataclass
-class InputDTMFFrame(DataFrame):
- """A DTMF button input"""
+class DTMFFrame(DataFrame):
+ """A DTMF button frame"""
button: KeypadEntry
+@dataclass
+class InputDTMFFrame(DTMFFrame):
+ """A DTMF button input"""
+
+ pass
+
+
+@dataclass
+class OutputDTMFFrame(DTMFFrame):
+ """A DTMF button output"""
+
+ pass
+
+
#
# System frames
#
@@ -410,11 +438,14 @@ class StartFrame(SystemFrame):
"""This is the first frame that should be pushed down a pipeline."""
clock: BaseClock
+ task_manager: TaskManager
+ audio_in_sample_rate: int = 16000
+ audio_out_sample_rate: int = 24000
allow_interruptions: bool = False
enable_metrics: bool = False
enable_usage_metrics: bool = False
- report_only_initial_ttfb: bool = False
observer: Optional["BaseObserver"] = None
+ report_only_initial_ttfb: bool = False
@dataclass
diff --git a/src/pipecat/pipeline/base_task.py b/src/pipecat/pipeline/base_task.py
new file mode 100644
index 000000000..b8622bac2
--- /dev/null
+++ b/src/pipecat/pipeline/base_task.py
@@ -0,0 +1,66 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import asyncio
+from abc import ABC, abstractmethod
+from typing import AsyncIterable, Iterable
+
+from pipecat.frames.frames import Frame
+
+
+class BaseTask(ABC):
+ @property
+ @abstractmethod
+ def id(self) -> int:
+ """Returns the unique indetifier for this task."""
+ pass
+
+ @property
+ @abstractmethod
+ def name(self) -> str:
+ """Returns the name of this task."""
+ pass
+
+ @abstractmethod
+ def set_event_loop(self, loop: asyncio.AbstractEventLoop):
+ """Sets the event loop that this task will run on."""
+ pass
+
+ @abstractmethod
+ def has_finished(self) -> bool:
+ """Indicates whether the tasks has finished. That is, all processors
+ have stopped.
+
+ """
+ pass
+
+ @abstractmethod
+ async def stop_when_done(self):
+ """This is a helper function that sends an EndFrame to the pipeline in
+ order to stop the task after everything in it has been processed.
+
+ """
+ pass
+
+ @abstractmethod
+ async def cancel(self):
+ """Stops the running pipeline immediately."""
+ pass
+
+ @abstractmethod
+ async def run(self):
+ """Starts running the given pipeline."""
+ pass
+
+ @abstractmethod
+ async def queue_frame(self, frame: Frame):
+ """Queue a frame to be pushed down the pipeline."""
+ pass
+
+ @abstractmethod
+ async def queue_frames(self, frames: Iterable[Frame] | AsyncIterable[Frame]):
+ """Queues multiple frames to be pushed down the pipeline."""
+ pass
diff --git a/src/pipecat/pipeline/parallel_pipeline.py b/src/pipecat/pipeline/parallel_pipeline.py
index 874a8c3a5..f77d09cc6 100644
--- a/src/pipecat/pipeline/parallel_pipeline.py
+++ b/src/pipecat/pipeline/parallel_pipeline.py
@@ -23,7 +23,7 @@ from pipecat.pipeline.pipeline import Pipeline
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
-class Source(FrameProcessor):
+class ParallelPipelineSource(FrameProcessor):
def __init__(
self,
upstream_queue: asyncio.Queue,
@@ -46,7 +46,7 @@ class Source(FrameProcessor):
await self.push_frame(frame, direction)
-class Sink(FrameProcessor):
+class ParallelPipelineSink(FrameProcessor):
def __init__(
self,
downstream_queue: asyncio.Queue,
@@ -92,8 +92,8 @@ class ParallelPipeline(BasePipeline):
raise TypeError(f"ParallelPipeline argument {processors} is not a list")
# We will add a source before the pipeline and a sink after.
- source = Source(self._up_queue, self._parallel_push_frame)
- sink = Sink(self._down_queue, self._parallel_push_frame)
+ source = ParallelPipelineSource(self._up_queue, self._parallel_push_frame)
+ sink = ParallelPipelineSink(self._down_queue, self._parallel_push_frame)
self._sources.append(source)
self._sinks.append(sink)
@@ -117,6 +117,7 @@ class ParallelPipeline(BasePipeline):
#
async def cleanup(self):
+ await super().cleanup()
await asyncio.gather(*[s.cleanup() for s in self._sources])
await asyncio.gather(*[p.cleanup() for p in self._pipelines])
await asyncio.gather(*[s.cleanup() for s in self._sinks])
@@ -150,22 +151,18 @@ class ParallelPipeline(BasePipeline):
async def _stop(self):
# The up task doesn't receive an EndFrame, so we just cancel it.
- self._up_task.cancel()
- await self._up_task
- # The down tasks waits for the last EndFrame send by the internal
+ await self.cancel_task(self._up_task)
+ # The down tasks waits for the last EndFrame sent by the internal
# pipelines.
await self._down_task
async def _cancel(self):
- self._up_task.cancel()
- await self._up_task
- self._down_task.cancel()
- await self._down_task
+ await self.cancel_task(self._up_task)
+ await self.cancel_task(self._down_task)
async def _create_tasks(self):
- loop = self.get_event_loop()
- self._up_task = loop.create_task(self._process_up_queue())
- self._down_task = loop.create_task(self._process_down_queue())
+ self._up_task = self.create_task(self._process_up_queue())
+ self._down_task = self.create_task(self._process_down_queue())
async def _drain_queues(self):
while not self._up_queue.empty:
@@ -185,32 +182,26 @@ class ParallelPipeline(BasePipeline):
async def _process_up_queue(self):
while True:
- try:
- frame = await self._up_queue.get()
- await self._parallel_push_frame(frame, FrameDirection.UPSTREAM)
- self._up_queue.task_done()
- except asyncio.CancelledError:
- break
+ frame = await self._up_queue.get()
+ await self._parallel_push_frame(frame, FrameDirection.UPSTREAM)
+ self._up_queue.task_done()
async def _process_down_queue(self):
running = True
while running:
- try:
- frame = await self._down_queue.get()
+ frame = await self._down_queue.get()
- endframe_counter = self._endframe_counter.get(frame.id, 0)
+ endframe_counter = self._endframe_counter.get(frame.id, 0)
- # If we have a counter, decrement it.
- if endframe_counter > 0:
- self._endframe_counter[frame.id] -= 1
- endframe_counter = self._endframe_counter[frame.id]
+ # If we have a counter, decrement it.
+ if endframe_counter > 0:
+ self._endframe_counter[frame.id] -= 1
+ endframe_counter = self._endframe_counter[frame.id]
- # If we don't have a counter or we reached 0, push the frame.
- if endframe_counter == 0:
- await self._parallel_push_frame(frame, FrameDirection.DOWNSTREAM)
+ # If we don't have a counter or we reached 0, push the frame.
+ if endframe_counter == 0:
+ await self._parallel_push_frame(frame, FrameDirection.DOWNSTREAM)
- running = not (endframe_counter == 0 and isinstance(frame, EndFrame))
+ running = not (endframe_counter == 0 and isinstance(frame, EndFrame))
- self._down_queue.task_done()
- except asyncio.CancelledError:
- break
+ self._down_queue.task_done()
diff --git a/src/pipecat/pipeline/pipeline.py b/src/pipecat/pipeline/pipeline.py
index e6b874ce9..270e100bd 100644
--- a/src/pipecat/pipeline/pipeline.py
+++ b/src/pipecat/pipeline/pipeline.py
@@ -71,6 +71,7 @@ class Pipeline(BasePipeline):
#
async def cleanup(self):
+ await super().cleanup()
await self._cleanup_processors()
async def process_frame(self, frame: Frame, direction: FrameDirection):
diff --git a/src/pipecat/pipeline/runner.py b/src/pipecat/pipeline/runner.py
index 34f7a0a8c..e74c491e8 100644
--- a/src/pipecat/pipeline/runner.py
+++ b/src/pipecat/pipeline/runner.py
@@ -5,7 +5,9 @@
#
import asyncio
+import gc
import signal
+from typing import Optional
from loguru import logger
@@ -14,11 +16,21 @@ from pipecat.utils.utils import obj_count, obj_id
class PipelineRunner:
- def __init__(self, *, name: str | None = None, handle_sigint: bool = True):
+ def __init__(
+ self,
+ *,
+ name: str | None = None,
+ handle_sigint: bool = True,
+ force_gc: bool = False,
+ loop: Optional[asyncio.AbstractEventLoop] = None,
+ ):
self.id: int = obj_id()
self.name: str = name or f"{self.__class__.__name__}#{obj_count(self)}"
self._tasks = {}
+ self._sig_task = None
+ self._force_gc = force_gc
+ self._loop = loop or asyncio.get_running_loop()
if handle_sigint:
self._setup_sigint()
@@ -26,8 +38,15 @@ class PipelineRunner:
async def run(self, task: PipelineTask):
logger.debug(f"Runner {self} started running {task}")
self._tasks[task.name] = task
+ task.set_event_loop(self._loop)
await task.run()
del self._tasks[task.name]
+ # If we are cancelling through a signal, make sure we wait for it so
+ # everything gets cleaned up nicely.
+ if self._sig_task:
+ await self._sig_task
+ if self._force_gc:
+ self._gc_collect()
logger.debug(f"Runner {self} finished running {task}")
async def stop_when_done(self):
@@ -40,16 +59,21 @@ class PipelineRunner:
def _setup_sigint(self):
loop = asyncio.get_running_loop()
- loop.add_signal_handler(
- signal.SIGINT, lambda *args: asyncio.create_task(self._sig_handler())
- )
- loop.add_signal_handler(
- signal.SIGTERM, lambda *args: asyncio.create_task(self._sig_handler())
- )
+ loop.add_signal_handler(signal.SIGINT, lambda *args: self._sig_handler())
+ loop.add_signal_handler(signal.SIGTERM, lambda *args: self._sig_handler())
- async def _sig_handler(self):
+ def _sig_handler(self):
+ if not self._sig_task:
+ self._sig_task = asyncio.create_task(self._sig_cancel())
+
+ async def _sig_cancel(self):
logger.warning(f"Interruption detected. Canceling runner {self}")
await self.cancel()
+ def _gc_collect(self):
+ collected = gc.collect()
+ logger.debug(f"Garbage collector: collected {collected} objects.")
+ logger.debug(f"Garbage collector: uncollectable objects {gc.garbage}")
+
def __str__(self):
return self.name
diff --git a/src/pipecat/pipeline/sync_parallel_pipeline.py b/src/pipecat/pipeline/sync_parallel_pipeline.py
index 78ddff074..2870a470d 100644
--- a/src/pipecat/pipeline/sync_parallel_pipeline.py
+++ b/src/pipecat/pipeline/sync_parallel_pipeline.py
@@ -24,7 +24,7 @@ class SyncFrame(ControlFrame):
pass
-class Source(FrameProcessor):
+class SyncParallelPipelineSource(FrameProcessor):
def __init__(self, upstream_queue: asyncio.Queue):
super().__init__()
self._up_queue = upstream_queue
@@ -39,7 +39,7 @@ class Source(FrameProcessor):
await self.push_frame(frame, direction)
-class Sink(FrameProcessor):
+class SyncParallelPipelineSink(FrameProcessor):
def __init__(self, downstream_queue: asyncio.Queue):
super().__init__()
self._down_queue = downstream_queue
@@ -76,18 +76,20 @@ class SyncParallelPipeline(BasePipeline):
# We add a source at the beginning of the pipeline and a sink at the end.
up_queue = asyncio.Queue()
down_queue = asyncio.Queue()
- source = Source(up_queue)
- sink = Sink(down_queue)
- processors: List[FrameProcessor] = [source] + processors + [sink]
+ source = SyncParallelPipelineSource(up_queue)
+ sink = SyncParallelPipelineSink(down_queue)
+
+ # Create pipeline
+ pipeline = Pipeline(processors)
+ source.link(pipeline)
+ pipeline.link(sink)
+ self._pipelines.append(pipeline)
# Keep track of sources and sinks. We also keep the output queue of
# the source and the sinks so we can use it later.
self._sources.append({"processor": source, "queue": down_queue})
self._sinks.append({"processor": sink, "queue": up_queue})
- # Create pipeline
- pipeline = Pipeline(processors)
- self._pipelines.append(pipeline)
logger.debug(f"Finished creating {self} pipelines")
#
@@ -101,6 +103,12 @@ class SyncParallelPipeline(BasePipeline):
# Frame processor
#
+ async def cleanup(self):
+ await super().cleanup()
+ await asyncio.gather(*[s["processor"].cleanup() for s in self._sources])
+ await asyncio.gather(*[p.cleanup() for p in self._pipelines])
+ await asyncio.gather(*[s["processor"].cleanup() for s in self._sinks])
+
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py
index 9263352e4..21b6af78b 100644
--- a/src/pipecat/pipeline/task.py
+++ b/src/pipecat/pipeline/task.py
@@ -5,7 +5,7 @@
#
import asyncio
-from typing import AsyncIterable, Iterable, List
+from typing import Any, AsyncIterable, Dict, Iterable, List
from loguru import logger
from pydantic import BaseModel, ConfigDict
@@ -27,8 +27,10 @@ from pipecat.frames.frames import (
from pipecat.metrics.metrics import ProcessingMetricsData, TTFBMetricsData
from pipecat.observers.base_observer import BaseObserver
from pipecat.pipeline.base_pipeline import BasePipeline
+from pipecat.pipeline.base_task import BaseTask
from pipecat.pipeline.task_observer import TaskObserver
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.utils.asyncio import TaskManager
from pipecat.utils.utils import obj_count, obj_id
HEARTBEAT_SECONDS = 1.0
@@ -39,15 +41,19 @@ class PipelineParams(BaseModel):
model_config = ConfigDict(arbitrary_types_allowed=True)
allow_interruptions: bool = False
+ audio_in_sample_rate: int = 16000
+ audio_out_sample_rate: int = 24000
enable_heartbeats: bool = False
enable_metrics: bool = False
enable_usage_metrics: bool = False
- send_initial_empty_metrics: bool = True
- report_only_initial_ttfb: bool = False
+ heartbeats_period_secs: float = HEARTBEAT_SECONDS
observers: List[BaseObserver] = []
+ report_only_initial_ttfb: bool = False
+ send_initial_empty_metrics: bool = True
+ start_metadata: Dict[str, Any] = {}
-class Source(FrameProcessor):
+class PipelineTaskSource(FrameProcessor):
"""This is the source processor that is linked at the beginning of the
pipeline given to the pipeline task. It allows us to easily push frames
downstream to the pipeline and also receive upstream frames coming from the
@@ -55,8 +61,8 @@ class Source(FrameProcessor):
"""
- def __init__(self, up_queue: asyncio.Queue):
- super().__init__()
+ def __init__(self, up_queue: asyncio.Queue, **kwargs):
+ super().__init__(**kwargs)
self._up_queue = up_queue
async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -69,15 +75,15 @@ class Source(FrameProcessor):
await self.push_frame(frame, direction)
-class Sink(FrameProcessor):
+class PipelineTaskSink(FrameProcessor):
"""This is the sink processor that is linked at the end of the pipeline
given to the pipeline task. It allows us to receive downstream frames and
act on them, for example, waiting to receive an EndFrame.
"""
- def __init__(self, down_queue: asyncio.Queue):
- super().__init__()
+ def __init__(self, down_queue: asyncio.Queue, **kwargs):
+ super().__init__(**kwargs)
self._down_queue = down_queue
async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -85,15 +91,15 @@ class Sink(FrameProcessor):
await self._down_queue.put(frame)
-class PipelineTask:
+class PipelineTask(BaseTask):
def __init__(
self,
pipeline: BasePipeline,
params: PipelineParams = PipelineParams(),
clock: BaseClock = SystemClock(),
):
- self.id: int = obj_id()
- self.name: str = f"{self.__class__.__name__}#{obj_count(self)}"
+ self._id: int = obj_id()
+ self._name: str = f"{self.__class__.__name__}#{obj_count(self)}"
self._pipeline = pipeline
self._clock = clock
@@ -113,15 +119,35 @@ class PipelineTask:
# down queue.
self._endframe_event = asyncio.Event()
- self._source = Source(self._up_queue)
+ self._source = PipelineTaskSource(self._up_queue)
self._source.link(pipeline)
- self._sink = Sink(self._down_queue)
+ self._sink = PipelineTaskSink(self._down_queue)
pipeline.link(self._sink)
- self._observer = TaskObserver(params.observers)
+ self._task_manager = TaskManager()
- def has_finished(self):
+ self._observer = TaskObserver(observers=params.observers, task_manager=self._task_manager)
+
+ @property
+ def id(self) -> int:
+ """Returns the unique indetifier for this task."""
+ return self._id
+
+ @property
+ def name(self) -> str:
+ """Returns the name of this task."""
+ return self._name
+
+ @property
+ def params(self) -> PipelineParams:
+ """Returns the pipeline parameters of this task."""
+ return self._params
+
+ def set_event_loop(self, loop: asyncio.AbstractEventLoop):
+ self._task_manager.set_event_loop(loop)
+
+ def has_finished(self) -> bool:
"""Indicates whether the tasks has finished. That is, all processors
have stopped.
@@ -145,14 +171,27 @@ class PipelineTask:
# out-of-band from the main streaming task which is what we want since
# we want to cancel right away.
await self._source.push_frame(CancelFrame())
- await self._cancel_tasks(True)
+ # Only cancel the push task. Everything else will be cancelled in run().
+ await self._task_manager.cancel_task(self._process_push_task)
async def run(self):
"""
Starts running the given pipeline.
"""
- tasks = self._create_tasks()
- await asyncio.gather(*tasks)
+ if self.has_finished():
+ return
+ try:
+ push_task = await self._create_tasks()
+ await self._task_manager.wait_for_task(push_task)
+ except asyncio.CancelledError:
+ # We are awaiting on the push task and it might be cancelled
+ # (e.g. Ctrl-C). This means we will get a CancelledError here as
+ # well, because you get a CancelledError in every place you are
+ # awaiting a task.
+ pass
+ await self._cancel_tasks()
+ await self._cleanup()
+ self._print_dangling_tasks()
self._finished = True
async def queue_frame(self, frame: Frame):
@@ -172,42 +211,42 @@ class PipelineTask:
for frame in frames:
await self.queue_frame(frame)
- def _create_tasks(self):
- tasks = []
- self._process_up_task = asyncio.create_task(self._process_up_queue())
- self._process_down_task = asyncio.create_task(self._process_down_queue())
- self._process_push_task = asyncio.create_task(self._process_push_queue())
+ async def _create_tasks(self):
+ self._process_up_task = self._task_manager.create_task(
+ self._process_up_queue(), f"{self}::_process_up_queue"
+ )
+ self._process_down_task = self._task_manager.create_task(
+ self._process_down_queue(), f"{self}::_process_down_queue"
+ )
+ self._process_push_task = self._task_manager.create_task(
+ self._process_push_queue(), f"{self}::_process_push_queue"
+ )
- tasks = [self._process_up_task, self._process_down_task, self._process_push_task]
+ await self._observer.start()
- return tasks
+ return self._process_push_task
def _maybe_start_heartbeat_tasks(self):
if self._params.enable_heartbeats:
- self._heartbeat_push_task = asyncio.create_task(self._heartbeat_push_handler())
- self._heartbeat_monitor_task = asyncio.create_task(self._heartbeat_monitor_handler())
+ self._heartbeat_push_task = self._task_manager.create_task(
+ self._heartbeat_push_handler(), f"{self}::_heartbeat_push_handler"
+ )
+ self._heartbeat_monitor_task = self._task_manager.create_task(
+ self._heartbeat_monitor_handler(), f"{self}::_heartbeat_monitor_handler"
+ )
- async def _cancel_tasks(self, cancel_push: bool):
+ async def _cancel_tasks(self):
await self._maybe_cancel_heartbeat_tasks()
- if cancel_push:
- self._process_push_task.cancel()
- await self._process_push_task
-
- self._process_up_task.cancel()
- await self._process_up_task
-
- self._process_down_task.cancel()
- await self._process_down_task
+ await self._task_manager.cancel_task(self._process_up_task)
+ await self._task_manager.cancel_task(self._process_down_task)
await self._observer.stop()
async def _maybe_cancel_heartbeat_tasks(self):
if self._params.enable_heartbeats:
- self._heartbeat_push_task.cancel()
- await self._heartbeat_push_task
- self._heartbeat_monitor_task.cancel()
- await self._heartbeat_monitor_task
+ await self._task_manager.cancel_task(self._heartbeat_push_task)
+ await self._task_manager.cancel_task(self._heartbeat_monitor_task)
def _initial_metrics_frame(self) -> MetricsFrame:
processors = self._pipeline.processors_with_metrics()
@@ -221,6 +260,11 @@ class PipelineTask:
await self._endframe_event.wait()
self._endframe_event.clear()
+ async def _cleanup(self):
+ await self._source.cleanup()
+ await self._pipeline.cleanup()
+ await self._sink.cleanup()
+
async def _process_push_queue(self):
"""This is the task that runs the pipeline for the first time by sending
a StartFrame and by pushing any other frames queued by the user. It runs
@@ -232,13 +276,17 @@ class PipelineTask:
self._maybe_start_heartbeat_tasks()
start_frame = StartFrame(
+ clock=self._clock,
+ task_manager=self._task_manager,
allow_interruptions=self._params.allow_interruptions,
+ audio_in_sample_rate=self._params.audio_in_sample_rate,
+ audio_out_sample_rate=self._params.audio_out_sample_rate,
enable_metrics=self._params.enable_metrics,
enable_usage_metrics=self._params.enable_usage_metrics,
- report_only_initial_ttfb=self._params.report_only_initial_ttfb,
observer=self._observer,
- clock=self._clock,
+ report_only_initial_ttfb=self._params.report_only_initial_ttfb,
)
+ start_frame.metadata = self._params.start_metadata
await self._source.queue_frame(start_frame, FrameDirection.DOWNSTREAM)
if self._params.enable_metrics and self._params.send_initial_empty_metrics:
@@ -247,24 +295,16 @@ class PipelineTask:
running = True
should_cleanup = True
while running:
- try:
- frame = await self._push_queue.get()
- await self._source.queue_frame(frame, FrameDirection.DOWNSTREAM)
- if isinstance(frame, EndFrame):
- await self._wait_for_endframe()
- running = not isinstance(frame, (StopTaskFrame, EndFrame))
- should_cleanup = not isinstance(frame, StopTaskFrame)
- self._push_queue.task_done()
- except asyncio.CancelledError:
- break
+ frame = await self._push_queue.get()
+ await self._source.queue_frame(frame, FrameDirection.DOWNSTREAM)
+ if isinstance(frame, EndFrame):
+ await self._wait_for_endframe()
+ running = not isinstance(frame, (CancelFrame, EndFrame, StopTaskFrame))
+ should_cleanup = not isinstance(frame, StopTaskFrame)
+ self._push_queue.task_done()
# Cleanup only if we need to.
if should_cleanup:
- await self._source.cleanup()
- await self._pipeline.cleanup()
- await self._sink.cleanup()
- # Finally, cancel internal tasks. We don't cancel the push tasks because
- # that's us.
- await self._cancel_tasks(False)
+ await self._cleanup()
async def _process_up_queue(self):
"""This is the task that processes frames coming upstream from the
@@ -274,26 +314,23 @@ class PipelineTask:
"""
while True:
- try:
- frame = await self._up_queue.get()
- if isinstance(frame, EndTaskFrame):
- # Tell the task we should end nicely.
- await self.queue_frame(EndFrame())
- elif isinstance(frame, CancelTaskFrame):
- # Tell the task we should end right away.
+ frame = await self._up_queue.get()
+ if isinstance(frame, EndTaskFrame):
+ # Tell the task we should end nicely.
+ await self.queue_frame(EndFrame())
+ elif isinstance(frame, CancelTaskFrame):
+ # Tell the task we should end right away.
+ await self.queue_frame(CancelFrame())
+ elif isinstance(frame, StopTaskFrame):
+ await self.queue_frame(StopTaskFrame())
+ elif isinstance(frame, ErrorFrame):
+ logger.error(f"Error running app: {frame}")
+ if frame.fatal:
+ # Cancel all tasks downstream.
await self.queue_frame(CancelFrame())
- elif isinstance(frame, StopTaskFrame):
+ # Tell the task we should stop.
await self.queue_frame(StopTaskFrame())
- elif isinstance(frame, ErrorFrame):
- logger.error(f"Error running app: {frame}")
- if frame.fatal:
- # Cancel all tasks downstream.
- await self.queue_frame(CancelFrame())
- # Tell the task we should stop.
- await self.queue_frame(StopTaskFrame())
- self._up_queue.task_done()
- except asyncio.CancelledError:
- break
+ self._up_queue.task_done()
async def _process_down_queue(self):
"""This tasks process frames coming downstream from the pipeline. For
@@ -303,29 +340,23 @@ class PipelineTask:
"""
while True:
- try:
- frame = await self._down_queue.get()
- if isinstance(frame, EndFrame):
- self._endframe_event.set()
- elif isinstance(frame, HeartbeatFrame):
- await self._heartbeat_queue.put(frame)
- self._down_queue.task_done()
- except asyncio.CancelledError:
- break
+ frame = await self._down_queue.get()
+ if isinstance(frame, EndFrame):
+ self._endframe_event.set()
+ elif isinstance(frame, HeartbeatFrame):
+ await self._heartbeat_queue.put(frame)
+ self._down_queue.task_done()
async def _heartbeat_push_handler(self):
"""
- This tasks pushes a heartbeat frame every HEARTBEAT_SECONDS.
+ This tasks pushes a heartbeat frame every heartbeat period.
"""
while True:
- try:
- # Don't use `queue_frame()` because if an EndFrame is queued the
- # task will just stop waiting for the pipeline to finish not
- # allowing more frames to be pushed.
- await self._source.queue_frame(HeartbeatFrame(timestamp=self._clock.get_time()))
- await asyncio.sleep(HEARTBEAT_SECONDS)
- except asyncio.CancelledError:
- break
+ # Don't use `queue_frame()` because if an EndFrame is queued the
+ # task will just stop waiting for the pipeline to finish not
+ # allowing more frames to be pushed.
+ await self._source.queue_frame(HeartbeatFrame(timestamp=self._clock.get_time()))
+ await asyncio.sleep(self._params.heartbeats_period_secs)
async def _heartbeat_monitor_handler(self):
"""This tasks monitors heartbeat frames. If a heartbeat frame has not
@@ -345,8 +376,11 @@ class PipelineTask:
logger.warning(
f"{self}: heartbeat frame not received for more than {wait_time} seconds"
)
- except asyncio.CancelledError:
- break
+
+ def _print_dangling_tasks(self):
+ tasks = [t.get_name() for t in self._task_manager.current_tasks()]
+ if tasks:
+ logger.warning(f"Dangling tasks detected: {tasks}")
def __str__(self):
return self.name
diff --git a/src/pipecat/pipeline/task_observer.py b/src/pipecat/pipeline/task_observer.py
index 2fd13f517..abcbf513b 100644
--- a/src/pipecat/pipeline/task_observer.py
+++ b/src/pipecat/pipeline/task_observer.py
@@ -12,6 +12,8 @@ from attr import dataclass
from pipecat.frames.frames import Frame
from pipecat.observers.base_observer import BaseObserver
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.utils.asyncio import TaskManager
+from pipecat.utils.utils import obj_count, obj_id
@dataclass
@@ -53,14 +55,29 @@ class TaskObserver(BaseObserver):
"""
- def __init__(self, observers: List[BaseObserver] = []):
- self._proxies: List[Proxy] = self._create_proxies(observers)
+ def __init__(self, *, observers: List[BaseObserver] = [], task_manager: TaskManager):
+ self._id: int = obj_id()
+ self._name: str = f"{self.__class__.__name__}#{obj_count(self)}"
+ self._observers = observers
+ self._task_manager = task_manager
+ self._proxies: List[Proxy] = []
+
+ @property
+ def id(self) -> int:
+ return self._id
+
+ @property
+ def name(self) -> str:
+ return self._name
+
+ async def start(self):
+ """Starts all proxy observer tasks."""
+ self._proxies = self._create_proxies(self._observers)
async def stop(self):
"""Stops all proxy observer tasks."""
for proxy in self._proxies:
- proxy.task.cancel()
- await proxy.task
+ await self._task_manager.cancel_task(proxy.task)
async def on_push_frame(
self,
@@ -81,17 +98,20 @@ class TaskObserver(BaseObserver):
proxies = []
for observer in observers:
queue = asyncio.Queue()
- task = asyncio.create_task(self._proxy_task_handler(queue, observer))
+ task = self._task_manager.create_task(
+ self._proxy_task_handler(queue, observer),
+ f"{self}::{observer.__class__.__name__}::_proxy_task_handler",
+ )
proxy = Proxy(queue=queue, task=task, observer=observer)
proxies.append(proxy)
return proxies
async def _proxy_task_handler(self, queue: asyncio.Queue, observer: BaseObserver):
while True:
- try:
- data = await queue.get()
- await observer.on_push_frame(
- data.src, data.dst, data.frame, data.direction, data.timestamp
- )
- except asyncio.CancelledError:
- break
+ data = await queue.get()
+ await observer.on_push_frame(
+ data.src, data.dst, data.frame, data.direction, data.timestamp
+ )
+
+ def __str__(self):
+ return self.name
diff --git a/src/pipecat/processors/aggregators/gated_openai_llm_context.py b/src/pipecat/processors/aggregators/gated_openai_llm_context.py
index a185528f2..f3ae78121 100644
--- a/src/pipecat/processors/aggregators/gated_openai_llm_context.py
+++ b/src/pipecat/processors/aggregators/gated_openai_llm_context.py
@@ -4,8 +4,6 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
-import asyncio
-
from pipecat.frames.frames import CancelFrame, EndFrame, Frame, StartFrame
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
@@ -18,9 +16,10 @@ class GatedOpenAILLMContextAggregator(FrameProcessor):
"""
- def __init__(self, notifier: BaseNotifier, **kwargs):
+ def __init__(self, *, notifier: BaseNotifier, start_open: bool = False, **kwargs):
super().__init__(**kwargs)
self._notifier = notifier
+ self._start_open = start_open
self._last_context_frame = None
async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -33,23 +32,23 @@ class GatedOpenAILLMContextAggregator(FrameProcessor):
await self._stop()
await self.push_frame(frame)
elif isinstance(frame, OpenAILLMContextFrame):
- self._last_context_frame = frame
+ if self._start_open:
+ self._start_open = False
+ await self.push_frame(frame, direction)
+ else:
+ self._last_context_frame = frame
else:
await self.push_frame(frame, direction)
async def _start(self):
- self._gate_task = self.get_event_loop().create_task(self._gate_task_handler())
+ self._gate_task = self.create_task(self._gate_task_handler())
async def _stop(self):
- self._gate_task.cancel()
- await self._gate_task
+ await self.cancel_task(self._gate_task)
async def _gate_task_handler(self):
while True:
- try:
- await self._notifier.wait()
- if self._last_context_frame:
- await self.push_frame(self._last_context_frame)
- self._last_context_frame = None
- except asyncio.CancelledError:
- break
+ await self._notifier.wait()
+ if self._last_context_frame:
+ await self.push_frame(self._last_context_frame)
+ self._last_context_frame = None
diff --git a/src/pipecat/processors/async_generator.py b/src/pipecat/processors/async_generator.py
index e13026cdb..142d9cb47 100644
--- a/src/pipecat/processors/async_generator.py
+++ b/src/pipecat/processors/async_generator.py
@@ -30,7 +30,7 @@ class AsyncGeneratorProcessor(FrameProcessor):
if isinstance(frame, (CancelFrame, EndFrame)):
await self._data_queue.put(None)
else:
- data = self._serializer.serialize(frame)
+ data = await self._serializer.serialize(frame)
if data:
await self._data_queue.put(data)
diff --git a/src/pipecat/processors/audio/audio_buffer_processor.py b/src/pipecat/processors/audio/audio_buffer_processor.py
index 34825ab20..a1f213180 100644
--- a/src/pipecat/processors/audio/audio_buffer_processor.py
+++ b/src/pipecat/processors/audio/audio_buffer_processor.py
@@ -4,11 +4,18 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
-from pipecat.audio.utils import interleave_stereo_audio, mix_audio, resample_audio
+import time
+from typing import Optional
+
+from pipecat.audio.utils import create_default_resampler, interleave_stereo_audio, mix_audio
from pipecat.frames.frames import (
+ AudioRawFrame,
+ CancelFrame,
+ EndFrame,
Frame,
InputAudioRawFrame,
OutputAudioRawFrame,
+ StartFrame,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
@@ -28,16 +35,29 @@ class AudioBufferProcessor(FrameProcessor):
"""
def __init__(
- self, *, sample_rate: int = 24000, num_channels: int = 1, buffer_size: int = 0, **kwargs
+ self,
+ *,
+ sample_rate: Optional[int] = None,
+ num_channels: int = 1,
+ buffer_size: int = 0,
+ **kwargs,
):
super().__init__(**kwargs)
- self._sample_rate = sample_rate
+ self._init_sample_rate = sample_rate
+ self._sample_rate = 0
self._num_channels = num_channels
self._buffer_size = buffer_size
self._user_audio_buffer = bytearray()
self._bot_audio_buffer = bytearray()
+ self._last_user_frame_at = 0
+ self._last_bot_frame_at = 0
+
+ self._recording = False
+
+ self._resampler = create_default_resampler()
+
self._register_event_handler("on_audio_data")
@property
@@ -63,40 +83,83 @@ class AudioBufferProcessor(FrameProcessor):
else:
return b""
- def reset_audio_buffers(self):
- self._user_audio_buffer = bytearray()
- self._bot_audio_buffer = bytearray()
+ async def start_recording(self):
+ self._recording = True
+ self._reset_recording()
+
+ async def stop_recording(self):
+ await self._call_on_audio_data_handler()
+ self._recording = False
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
- # Include all audio from the user.
- if isinstance(frame, InputAudioRawFrame):
- resampled = resample_audio(frame.audio, frame.sample_rate, self._sample_rate)
+ # Update output sample rate if necessary.
+ if isinstance(frame, StartFrame):
+ self._update_sample_rate(frame)
+
+ if self._recording and isinstance(frame, InputAudioRawFrame):
+ # Add silence if we need to.
+ silence = self._compute_silence(self._last_user_frame_at)
+ self._user_audio_buffer.extend(silence)
+ # Add user audio.
+ resampled = await self._resample_audio(frame)
self._user_audio_buffer.extend(resampled)
- # Sync the bot's buffer to the user's buffer by adding silence if needed
- if len(self._user_audio_buffer) > len(self._bot_audio_buffer):
- silence = b"\x00" * len(resampled)
- self._bot_audio_buffer.extend(silence)
- # If the bot is speaking, include all audio from the bot.
- elif isinstance(frame, OutputAudioRawFrame):
- resampled = resample_audio(frame.audio, frame.sample_rate, self._sample_rate)
+ # Save time of frame so we can compute silence.
+ self._last_user_frame_at = time.time()
+ elif self._recording and isinstance(frame, OutputAudioRawFrame):
+ # Add silence if we need to.
+ silence = self._compute_silence(self._last_bot_frame_at)
+ self._bot_audio_buffer.extend(silence)
+ # Add bot audio.
+ resampled = await self._resample_audio(frame)
self._bot_audio_buffer.extend(resampled)
+ # Save time of frame so we can compute silence.
+ self._last_bot_frame_at = time.time()
if self._buffer_size > 0 and len(self._user_audio_buffer) > self._buffer_size:
await self._call_on_audio_data_handler()
+ if isinstance(frame, (CancelFrame, EndFrame)):
+ await self.stop_recording()
+
await self.push_frame(frame, direction)
+ def _update_sample_rate(self, frame: StartFrame):
+ self._sample_rate = self._init_sample_rate or frame.audio_out_sample_rate
+
async def _call_on_audio_data_handler(self):
- if not self.has_audio():
+ if not self.has_audio() or not self._recording:
return
merged_audio = self.merge_audio_buffers()
await self._call_event_handler(
"on_audio_data", merged_audio, self._sample_rate, self._num_channels
)
- self.reset_audio_buffers()
+ self._reset_audio_buffers()
def _buffer_has_audio(self, buffer: bytearray) -> bool:
return buffer is not None and len(buffer) > 0
+
+ def _reset_recording(self):
+ self._reset_audio_buffers()
+ self._last_user_frame_at = time.time()
+ self._last_bot_frame_at = time.time()
+
+ def _reset_audio_buffers(self):
+ self._user_audio_buffer = bytearray()
+ self._bot_audio_buffer = bytearray()
+
+ async def _resample_audio(self, frame: AudioRawFrame) -> bytes:
+ return await self._resampler.resample(frame.audio, frame.sample_rate, self._sample_rate)
+
+ def _compute_silence(self, from_time: float) -> bytes:
+ quiet_time = time.time() - from_time
+ # We should get audio frames very frequently. We pick 100ms because
+ # that's big enough, but it could be even a bit slower since we usually
+ # do 20ms audio frames.
+ if from_time == 0 or quiet_time < 0.1:
+ return b""
+ num_bytes = int(quiet_time * self._sample_rate) * 2
+ silence = b"\x00" * num_bytes
+ return silence
diff --git a/src/pipecat/processors/audio/vad/silero.py b/src/pipecat/processors/audio/vad/silero.py
index 2d6619a0c..edfe484ba 100644
--- a/src/pipecat/processors/audio/vad/silero.py
+++ b/src/pipecat/processors/audio/vad/silero.py
@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
+from typing import Optional
+
from loguru import logger
from pipecat.audio.vad.silero import SileroVADAnalyzer
@@ -11,6 +13,7 @@ from pipecat.audio.vad.vad_analyzer import VADParams, VADState
from pipecat.frames.frames import (
AudioRawFrame,
Frame,
+ StartFrame,
StartInterruptionFrame,
StopInterruptionFrame,
UserStartedSpeakingFrame,
@@ -23,7 +26,7 @@ class SileroVAD(FrameProcessor):
def __init__(
self,
*,
- sample_rate: int = 16000,
+ sample_rate: Optional[int] = None,
vad_params: VADParams = VADParams(),
audio_passthrough: bool = False,
):
@@ -41,6 +44,9 @@ class SileroVAD(FrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
+ if isinstance(frame, StartFrame):
+ self._vad_analyzer.set_sample_rate(frame.audio_in_sample_rate)
+
if isinstance(frame, AudioRawFrame):
await self._analyze_audio(frame)
if self._audio_passthrough:
diff --git a/src/pipecat/processors/filters/frame_filter.py b/src/pipecat/processors/filters/frame_filter.py
index a2fb90505..43e2dab95 100644
--- a/src/pipecat/processors/filters/frame_filter.py
+++ b/src/pipecat/processors/filters/frame_filter.py
@@ -11,7 +11,7 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class FrameFilter(FrameProcessor):
- def __init__(self, types: Tuple[Type[Frame]]):
+ def __init__(self, types: Tuple[Type[Frame], ...]):
super().__init__()
self._types = types
diff --git a/src/pipecat/processors/filters/stt_mute_filter.py b/src/pipecat/processors/filters/stt_mute_filter.py
index 65f224fc1..1a276df18 100644
--- a/src/pipecat/processors/filters/stt_mute_filter.py
+++ b/src/pipecat/processors/filters/stt_mute_filter.py
@@ -121,6 +121,8 @@ class STTMuteFilter(FrameProcessor):
return False
async def process_frame(self, frame: Frame, direction: FrameDirection):
+ await super().process_frame(frame, direction)
+
"""Processes incoming frames and manages muting state."""
# Handle function call state changes
if isinstance(frame, FunctionCallInProgressFrame):
diff --git a/src/pipecat/processors/frame_processor.py b/src/pipecat/processors/frame_processor.py
index ae830c52c..260e53ef2 100644
--- a/src/pipecat/processors/frame_processor.py
+++ b/src/pipecat/processors/frame_processor.py
@@ -7,14 +7,13 @@
import asyncio
import inspect
from enum import Enum
-from typing import Awaitable, Callable, Optional
+from typing import Awaitable, Callable, Coroutine, Optional
from loguru import logger
from pipecat.clocks.base_clock import BaseClock
from pipecat.frames.frames import (
CancelFrame,
- EndFrame,
ErrorFrame,
Frame,
StartFrame,
@@ -24,6 +23,7 @@ from pipecat.frames.frames import (
)
from pipecat.metrics.metrics import LLMTokenUsage, MetricsData
from pipecat.processors.metrics.frame_processor_metrics import FrameProcessorMetrics
+from pipecat.utils.asyncio import TaskManager
from pipecat.utils.utils import obj_count, obj_id
@@ -36,24 +36,25 @@ class FrameProcessor:
def __init__(
self,
*,
- name: str | None = None,
- metrics: FrameProcessorMetrics | None = None,
- loop: asyncio.AbstractEventLoop | None = None,
+ name: Optional[str] = None,
+ metrics: Optional[FrameProcessorMetrics] = None,
**kwargs,
):
- self.id: int = obj_id()
- self.name = name or f"{self.__class__.__name__}#{obj_count(self)}"
- self._parent: "FrameProcessor" | None = None
- self._prev: "FrameProcessor" | None = None
- self._next: "FrameProcessor" | None = None
- self._loop: asyncio.AbstractEventLoop = loop or asyncio.get_running_loop()
+ self._id: int = obj_id()
+ self._name = name or f"{self.__class__.__name__}#{obj_count(self)}"
+ self._parent: Optional["FrameProcessor"] = None
+ self._prev: Optional["FrameProcessor"] = None
+ self._next: Optional["FrameProcessor"] = None
self._event_handlers: dict = {}
# Clock
- self._clock: BaseClock | None = None
+ self._clock: Optional[BaseClock] = None
- # Properties
+ # Task Manager
+ self._task_manager: Optional[TaskManager] = None
+
+ # Other properties
self._allow_interruptions = False
self._enable_metrics = False
self._enable_usage_metrics = False
@@ -72,16 +73,24 @@ class FrameProcessor:
self._metrics.set_processor_name(self.name)
# Processors have an input queue. The input queue will be processed
- # immediately (default) or it will block if `pause_processing_frames()`
- # is called. To resume processing frames we need to call
+ # immediately (default) or it will block if `pause_processing_frames()` is
+ # called. To resume processing frames we need to call
# `resume_processing_frames()`.
self.__should_block_frames = False
- self.__create_input_task()
+ self.__input_frame_task: Optional[asyncio.Task] = None
# Every processor in Pipecat should only output frames from a single
- # task. This avoid problems like audio overlapping. System frames are
- # the exception to this rule. This create this task.
- self.__create_push_task()
+ # task. This avoid problems like audio overlapping. System frames are the
+ # exception to this rule. This create this task.
+ self.__push_frame_task: Optional[asyncio.Task] = None
+
+ @property
+ def id(self) -> int:
+ return self._id
+
+ @property
+ def name(self) -> str:
+ return self._name
@property
def interruptions_allowed(self):
@@ -141,6 +150,22 @@ class FrameProcessor:
await self.stop_ttfb_metrics()
await self.stop_processing_metrics()
+ def create_task(self, coroutine: Coroutine) -> asyncio.Task:
+ if not self._task_manager:
+ raise Exception(f"{self} TaskManager is still not initialized.")
+ name = f"{self}::{coroutine.cr_code.co_name}"
+ return self._task_manager.create_task(coroutine, name)
+
+ async def cancel_task(self, task: asyncio.Task, timeout: Optional[float] = None):
+ if not self._task_manager:
+ raise Exception(f"{self} TaskManager is still not initialized.")
+ await self._task_manager.cancel_task(task, timeout)
+
+ async def wait_for_task(self, task: asyncio.Task, timeout: Optional[float] = None):
+ if not self._task_manager:
+ raise Exception(f"{self} TaskManager is still not initialized.")
+ await self._task_manager.wait_for_task(task, timeout)
+
async def cleanup(self):
await self.__cancel_input_task()
await self.__cancel_push_task()
@@ -151,17 +176,26 @@ class FrameProcessor:
logger.debug(f"Linking {self} -> {self._next}")
def get_event_loop(self) -> asyncio.AbstractEventLoop:
- return self._loop
+ if not self._task_manager:
+ raise Exception(f"{self} TaskManager is still not initialized.")
+ return self._task_manager.get_event_loop()
def set_parent(self, parent: "FrameProcessor"):
self._parent = parent
- def get_parent(self) -> "FrameProcessor":
+ def get_parent(self) -> Optional["FrameProcessor"]:
return self._parent
def get_clock(self) -> BaseClock:
+ if not self._clock:
+ raise Exception(f"{self} Clock is still not initialized.")
return self._clock
+ def get_task_manager(self) -> TaskManager:
+ if not self._task_manager:
+ raise Exception(f"{self} TaskManager is still not initialized.")
+ return self._task_manager
+
async def queue_frame(
self,
frame: Frame,
@@ -186,18 +220,19 @@ class FrameProcessor:
self.__should_block_frames = True
async def resume_processing_frames(self):
- logger.trace("f{self}: resuming frame processing")
+ logger.trace(f"{self}: resuming frame processing")
self.__input_event.set()
- self.__should_block_frames = False
async def process_frame(self, frame: Frame, direction: FrameDirection):
if isinstance(frame, StartFrame):
self._clock = frame.clock
+ self._task_manager = frame.task_manager
self._allow_interruptions = frame.allow_interruptions
self._enable_metrics = frame.enable_metrics
self._enable_usage_metrics = frame.enable_usage_metrics
self._report_only_initial_ttfb = frame.report_only_initial_ttfb
self._observer = frame.observer
+ await self.__start(frame)
elif isinstance(frame, StartInterruptionFrame):
await self._start_interruption()
await self.stop_all_metrics()
@@ -232,6 +267,10 @@ class FrameProcessor:
raise Exception(f"Event handler {event_name} already registered")
self._event_handlers[event_name] = []
+ async def __start(self, frame: StartFrame):
+ self.__create_input_task()
+ self.__create_push_task()
+
#
# Handle interruptions
#
@@ -281,68 +320,52 @@ class FrameProcessor:
raise
def __create_input_task(self):
- self.__should_block_frames = False
- self.__input_queue = asyncio.Queue()
- self.__input_frame_task = self.get_event_loop().create_task(
- self.__input_frame_task_handler()
- )
- self.__input_event = asyncio.Event()
+ if not self.__input_frame_task:
+ self.__should_block_frames = False
+ self.__input_queue = asyncio.Queue()
+ self.__input_event = asyncio.Event()
+ self.__input_frame_task = self.create_task(self.__input_frame_task_handler())
async def __cancel_input_task(self):
- self.__input_frame_task.cancel()
- await self.__input_frame_task
+ if self.__input_frame_task:
+ await self.cancel_task(self.__input_frame_task)
+ self.__input_frame_task = None
async def __input_frame_task_handler(self):
- running = True
- while running:
- try:
- if self.__should_block_frames:
- logger.trace(f"{self}: frame processing paused")
- await self.__input_event.wait()
- self.__input_event.clear()
- logger.trace(f"{self}: frame processing resumed")
+ while True:
+ if self.__should_block_frames:
+ logger.trace(f"{self}: frame processing paused")
+ await self.__input_event.wait()
+ self.__input_event.clear()
+ self.__should_block_frames = False
+ logger.trace(f"{self}: frame processing resumed")
- (frame, direction, callback) = await self.__input_queue.get()
+ (frame, direction, callback) = await self.__input_queue.get()
- # Process the frame.
- await self.process_frame(frame, direction)
+ # Process the frame.
+ await self.process_frame(frame, direction)
- # If this frame has an associated callback, call it now.
- if callback:
- await callback(self, frame, direction)
+ # If this frame has an associated callback, call it now.
+ if callback:
+ await callback(self, frame, direction)
- running = not isinstance(frame, EndFrame)
-
- self.__input_queue.task_done()
- except asyncio.CancelledError:
- logger.trace(f"{self}: cancelled input task")
- break
- except Exception as e:
- logger.exception(f"{self}: Uncaught exception {e}")
- await self.push_error(ErrorFrame(str(e)))
+ self.__input_queue.task_done()
def __create_push_task(self):
- self.__push_queue = asyncio.Queue()
- self.__push_frame_task = self.get_event_loop().create_task(self.__push_frame_task_handler())
+ if not self.__push_frame_task:
+ self.__push_queue = asyncio.Queue()
+ self.__push_frame_task = self.create_task(self.__push_frame_task_handler())
async def __cancel_push_task(self):
- self.__push_frame_task.cancel()
- await self.__push_frame_task
+ if self.__push_frame_task:
+ await self.cancel_task(self.__push_frame_task)
+ self.__push_frame_task = None
async def __push_frame_task_handler(self):
- running = True
- while running:
- try:
- (frame, direction) = await self.__push_queue.get()
- await self.__internal_push_frame(frame, direction)
- running = not isinstance(frame, EndFrame)
- self.__push_queue.task_done()
- except asyncio.CancelledError:
- logger.trace(f"{self}: cancelled push task")
- break
- except Exception as e:
- logger.exception(f"{self}: Uncaught exception {e}")
- await self.push_error(ErrorFrame(str(e)))
+ while True:
+ (frame, direction) = await self.__push_queue.get()
+ await self.__internal_push_frame(frame, direction)
+ self.__push_queue.task_done()
async def _call_event_handler(self, event_name: str, *args, **kwargs):
try:
diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py
index a689b93ad..90495f8c7 100644
--- a/src/pipecat/processors/frameworks/rtvi.py
+++ b/src/pipecat/processors/frameworks/rtvi.py
@@ -58,10 +58,14 @@ from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContextFrame,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.services.google.frames import LLMSearchOrigin, LLMSearchResponseFrame
from pipecat.utils.string import match_endofsentence
RTVI_PROTOCOL_VERSION = "0.3.0"
+RTVI_MESSAGE_LABEL = "rtvi-ai"
+RTVIMessageLiteral = Literal["rtvi-ai"]
+
ActionResult = Union[bool, int, float, str, list, dict]
@@ -154,7 +158,7 @@ class RTVIActionFrame(DataFrame):
class RTVIMessage(BaseModel):
- label: Literal["rtvi-ai"] = "rtvi-ai"
+ label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: str
id: str
data: Optional[Dict[str, Any]] = None
@@ -170,7 +174,7 @@ class RTVIErrorResponseData(BaseModel):
class RTVIErrorResponse(BaseModel):
- label: Literal["rtvi-ai"] = "rtvi-ai"
+ label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["error-response"] = "error-response"
id: str
data: RTVIErrorResponseData
@@ -182,7 +186,7 @@ class RTVIErrorData(BaseModel):
class RTVIError(BaseModel):
- label: Literal["rtvi-ai"] = "rtvi-ai"
+ label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["error"] = "error"
data: RTVIErrorData
@@ -192,7 +196,7 @@ class RTVIDescribeConfigData(BaseModel):
class RTVIDescribeConfig(BaseModel):
- label: Literal["rtvi-ai"] = "rtvi-ai"
+ label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["config-available"] = "config-available"
id: str
data: RTVIDescribeConfigData
@@ -203,14 +207,14 @@ class RTVIDescribeActionsData(BaseModel):
class RTVIDescribeActions(BaseModel):
- label: Literal["rtvi-ai"] = "rtvi-ai"
+ label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["actions-available"] = "actions-available"
id: str
data: RTVIDescribeActionsData
class RTVIConfigResponse(BaseModel):
- label: Literal["rtvi-ai"] = "rtvi-ai"
+ label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["config"] = "config"
id: str
data: RTVIConfig
@@ -221,7 +225,7 @@ class RTVIActionResponseData(BaseModel):
class RTVIActionResponse(BaseModel):
- label: Literal["rtvi-ai"] = "rtvi-ai"
+ label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["action-response"] = "action-response"
id: str
data: RTVIActionResponseData
@@ -233,7 +237,7 @@ class RTVIBotReadyData(BaseModel):
class RTVIBotReady(BaseModel):
- label: Literal["rtvi-ai"] = "rtvi-ai"
+ label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-ready"] = "bot-ready"
id: str
data: RTVIBotReadyData
@@ -246,7 +250,7 @@ class RTVILLMFunctionCallMessageData(BaseModel):
class RTVILLMFunctionCallMessage(BaseModel):
- label: Literal["rtvi-ai"] = "rtvi-ai"
+ label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["llm-function-call"] = "llm-function-call"
data: RTVILLMFunctionCallMessageData
@@ -256,7 +260,7 @@ class RTVILLMFunctionCallStartMessageData(BaseModel):
class RTVILLMFunctionCallStartMessage(BaseModel):
- label: Literal["rtvi-ai"] = "rtvi-ai"
+ label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["llm-function-call-start"] = "llm-function-call-start"
data: RTVILLMFunctionCallStartMessageData
@@ -269,22 +273,22 @@ class RTVILLMFunctionCallResultData(BaseModel):
class RTVIBotLLMStartedMessage(BaseModel):
- label: Literal["rtvi-ai"] = "rtvi-ai"
+ label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-llm-started"] = "bot-llm-started"
class RTVIBotLLMStoppedMessage(BaseModel):
- label: Literal["rtvi-ai"] = "rtvi-ai"
+ label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-llm-stopped"] = "bot-llm-stopped"
class RTVIBotTTSStartedMessage(BaseModel):
- label: Literal["rtvi-ai"] = "rtvi-ai"
+ label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-tts-started"] = "bot-tts-started"
class RTVIBotTTSStoppedMessage(BaseModel):
- label: Literal["rtvi-ai"] = "rtvi-ai"
+ label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-tts-stopped"] = "bot-tts-stopped"
@@ -292,20 +296,32 @@ class RTVITextMessageData(BaseModel):
text: str
+class RTVISearchResponseMessageData(BaseModel):
+ search_result: Optional[str]
+ rendered_content: Optional[str]
+ origins: List[LLMSearchOrigin]
+
+
class RTVIBotTranscriptionMessage(BaseModel):
- label: Literal["rtvi-ai"] = "rtvi-ai"
+ label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-transcription"] = "bot-transcription"
data: RTVITextMessageData
class RTVIBotLLMTextMessage(BaseModel):
- label: Literal["rtvi-ai"] = "rtvi-ai"
+ label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-llm-text"] = "bot-llm-text"
data: RTVITextMessageData
-class RTVIBotTTSTextMessage(BaseModel):
+class RTVIBotLLMSearchResponseMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
+ type: Literal["bot-llm-search-response"] = "bot-llm-search-response"
+ data: RTVISearchResponseMessageData
+
+
+class RTVIBotTTSTextMessage(BaseModel):
+ label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-tts-text"] = "bot-tts-text"
data: RTVITextMessageData
@@ -317,7 +333,7 @@ class RTVIAudioMessageData(BaseModel):
class RTVIBotTTSAudioMessage(BaseModel):
- label: Literal["rtvi-ai"] = "rtvi-ai"
+ label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-tts-audio"] = "bot-tts-audio"
data: RTVIAudioMessageData
@@ -330,39 +346,39 @@ class RTVIUserTranscriptionMessageData(BaseModel):
class RTVIUserTranscriptionMessage(BaseModel):
- label: Literal["rtvi-ai"] = "rtvi-ai"
+ label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["user-transcription"] = "user-transcription"
data: RTVIUserTranscriptionMessageData
class RTVIUserLLMTextMessage(BaseModel):
- label: Literal["rtvi-ai"] = "rtvi-ai"
+ label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["user-llm-text"] = "user-llm-text"
data: RTVITextMessageData
class RTVIUserStartedSpeakingMessage(BaseModel):
- label: Literal["rtvi-ai"] = "rtvi-ai"
+ label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["user-started-speaking"] = "user-started-speaking"
class RTVIUserStoppedSpeakingMessage(BaseModel):
- label: Literal["rtvi-ai"] = "rtvi-ai"
+ label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["user-stopped-speaking"] = "user-stopped-speaking"
class RTVIBotStartedSpeakingMessage(BaseModel):
- label: Literal["rtvi-ai"] = "rtvi-ai"
+ label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-started-speaking"] = "bot-started-speaking"
class RTVIBotStoppedSpeakingMessage(BaseModel):
- label: Literal["rtvi-ai"] = "rtvi-ai"
+ label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-stopped-speaking"] = "bot-stopped-speaking"
class RTVIMetricsMessage(BaseModel):
- label: Literal["rtvi-ai"] = "rtvi-ai"
+ label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["metrics"] = "metrics"
data: Mapping[str, Any]
@@ -607,6 +623,8 @@ class RTVIObserver(BaseObserver):
await self._push_transport_message_urgent(RTVIBotLLMStoppedMessage())
elif isinstance(frame, LLMTextFrame):
await self._handle_llm_text_frame(frame)
+ elif isinstance(frame, LLMSearchResponseFrame):
+ await self._handle_llm_search_response_frame(frame)
elif isinstance(frame, TTSStartedFrame):
await self._push_transport_message_urgent(RTVIBotTTSStartedMessage())
elif isinstance(frame, TTSStoppedFrame):
@@ -657,6 +675,16 @@ class RTVIObserver(BaseObserver):
if match_endofsentence(self._bot_transcription):
await self._push_bot_transcription()
+ async def _handle_llm_search_response_frame(self, frame: LLMSearchResponseFrame):
+ message = RTVIBotLLMSearchResponseMessage(
+ data=RTVISearchResponseMessageData(
+ search_result=frame.search_result,
+ origins=frame.origins,
+ rendered_content=frame.rendered_content,
+ )
+ )
+ await self._push_transport_message_urgent(message)
+
async def _handle_user_transcriptions(self, frame: Frame):
message = None
if isinstance(frame, TranscriptionFrame):
@@ -676,17 +704,20 @@ class RTVIObserver(BaseObserver):
await self._push_transport_message_urgent(message)
async def _handle_context(self, frame: OpenAILLMContextFrame):
- messages = frame.context.messages
- if len(messages) > 0:
- message = messages[-1]
- if message["role"] == "user":
- content = message["content"]
- if isinstance(content, list):
- text = " ".join(item["text"] for item in content if "text" in item)
- else:
- text = content
- rtvi_message = RTVIUserLLMTextMessage(data=RTVITextMessageData(text=text))
- await self._push_transport_message_urgent(rtvi_message)
+ try:
+ messages = frame.context.messages
+ if len(messages) > 0:
+ message = messages[-1]
+ if message["role"] == "user":
+ content = message["content"]
+ if isinstance(content, list):
+ text = " ".join(item["text"] for item in content if "text" in item)
+ else:
+ text = content
+ rtvi_message = RTVIUserLLMTextMessage(data=RTVITextMessageData(text=text))
+ await self._push_transport_message_urgent(rtvi_message)
+ except TypeError as e:
+ logger.warning(f"Caught an error while trying to handle context: {e}")
async def _handle_metrics(self, frame: MetricsFrame):
metrics = {}
@@ -733,11 +764,11 @@ class RTVIProcessor(FrameProcessor):
# A task to process incoming action frames.
self._action_queue = asyncio.Queue()
- self._action_task = self.get_event_loop().create_task(self._action_task_handler())
+ self._action_task: Optional[asyncio.Task] = None
# A task to process incoming transport messages.
self._message_queue = asyncio.Queue()
- self._message_task = self.get_event_loop().create_task(self._message_task_handler())
+ self._message_task: Optional[asyncio.Task] = None
self._register_event_handler("on_bot_started")
self._register_event_handler("on_client_ready")
@@ -832,6 +863,8 @@ class RTVIProcessor(FrameProcessor):
await self._pipeline.cleanup()
async def _start(self, frame: StartFrame):
+ self._action_task = self.create_task(self._action_task_handler())
+ self._message_task = self.create_task(self._message_task_handler())
await self._call_event_handler("on_bot_started")
async def _stop(self, frame: EndFrame):
@@ -842,13 +875,11 @@ class RTVIProcessor(FrameProcessor):
async def _cancel_tasks(self):
if self._action_task:
- self._action_task.cancel()
- await self._action_task
+ await self.cancel_task(self._action_task)
self._action_task = None
if self._message_task:
- self._message_task.cancel()
- await self._message_task
+ await self.cancel_task(self._message_task)
self._message_task = None
async def _push_transport_message(self, model: BaseModel, exclude_none: bool = True):
@@ -857,25 +888,23 @@ class RTVIProcessor(FrameProcessor):
async def _action_task_handler(self):
while True:
- try:
- frame = await self._action_queue.get()
- await self._handle_action(frame.message_id, frame.rtvi_action_run)
- self._action_queue.task_done()
- except asyncio.CancelledError:
- break
+ frame = await self._action_queue.get()
+ await self._handle_action(frame.message_id, frame.rtvi_action_run)
+ self._action_queue.task_done()
async def _message_task_handler(self):
while True:
- try:
- message = await self._message_queue.get()
- await self._handle_message(message)
- self._message_queue.task_done()
- except asyncio.CancelledError:
- break
+ message = await self._message_queue.get()
+ await self._handle_message(message)
+ self._message_queue.task_done()
async def _handle_transport_message(self, frame: TransportMessageUrgentFrame):
try:
- message = RTVIMessage.model_validate(frame.message)
+ transport_message = frame.message
+ if transport_message.get("label") != RTVI_MESSAGE_LABEL:
+ logger.warning(f"Ignoring not RTVI message: {transport_message}")
+ return
+ message = RTVIMessage.model_validate(transport_message)
await self._message_queue.put(message)
except ValidationError as e:
await self.send_error(f"Invalid RTVI transport message: {e}")
diff --git a/src/pipecat/processors/gstreamer/pipeline_source.py b/src/pipecat/processors/gstreamer/pipeline_source.py
index 125f46d76..12b9cef67 100644
--- a/src/pipecat/processors/gstreamer/pipeline_source.py
+++ b/src/pipecat/processors/gstreamer/pipeline_source.py
@@ -5,6 +5,7 @@
#
import asyncio
+from typing import Optional
from loguru import logger
from pydantic import BaseModel
@@ -38,7 +39,7 @@ class GStreamerPipelineSource(FrameProcessor):
class OutputParams(BaseModel):
video_width: int = 1280
video_height: int = 720
- audio_sample_rate: int = 24000
+ audio_sample_rate: Optional[int] = None
audio_channels: int = 1
clock_sync: bool = True
@@ -46,6 +47,7 @@ class GStreamerPipelineSource(FrameProcessor):
super().__init__(**kwargs)
self._out_params = out_params
+ self._sample_rate = 0
Gst.init()
@@ -90,6 +92,7 @@ class GStreamerPipelineSource(FrameProcessor):
await self.push_frame(frame, direction)
async def _start(self, frame: StartFrame):
+ self._sample_rate = self._out_params.audio_sample_rate or frame.audio_out_sample_rate
self._player.set_state(Gst.State.PLAYING)
async def _stop(self, frame: EndFrame):
@@ -122,7 +125,7 @@ class GStreamerPipelineSource(FrameProcessor):
audioresample = Gst.ElementFactory.make("audioresample", None)
audiocapsfilter = Gst.ElementFactory.make("capsfilter", None)
audiocaps = Gst.Caps.from_string(
- f"audio/x-raw,format=S16LE,rate={self._out_params.audio_sample_rate},channels={self._out_params.audio_channels},layout=interleaved"
+ f"audio/x-raw,format=S16LE,rate={self._sample_rate},channels={self._out_params.audio_channels},layout=interleaved"
)
audiocapsfilter.set_property("caps", audiocaps)
appsink_audio = Gst.ElementFactory.make("appsink", None)
@@ -188,7 +191,7 @@ class GStreamerPipelineSource(FrameProcessor):
(_, info) = buffer.map(Gst.MapFlags.READ)
frame = OutputAudioRawFrame(
audio=info.data,
- sample_rate=self._out_params.audio_sample_rate,
+ sample_rate=self._sample_rate,
num_channels=self._out_params.audio_channels,
)
asyncio.run_coroutine_threadsafe(self.push_frame(frame), self.get_event_loop())
diff --git a/src/pipecat/processors/idle_frame_processor.py b/src/pipecat/processors/idle_frame_processor.py
index 3f5f51e45..85ea215cf 100644
--- a/src/pipecat/processors/idle_frame_processor.py
+++ b/src/pipecat/processors/idle_frame_processor.py
@@ -49,12 +49,11 @@ class IdleFrameProcessor(FrameProcessor):
self._idle_event.set()
async def cleanup(self):
- self._idle_task.cancel()
- await self._idle_task
+ await self.cancel_task(self._idle_task)
def _create_idle_task(self):
self._idle_event = asyncio.Event()
- self._idle_task = self.get_event_loop().create_task(self._idle_task_handler())
+ self._idle_task = self.create_task(self._idle_task_handler())
async def _idle_task_handler(self):
while True:
@@ -62,7 +61,5 @@ class IdleFrameProcessor(FrameProcessor):
await asyncio.wait_for(self._idle_event.wait(), timeout=self._timeout)
except asyncio.TimeoutError:
await self._callback(self)
- except asyncio.CancelledError:
- break
finally:
self._idle_event.clear()
diff --git a/src/pipecat/processors/logger.py b/src/pipecat/processors/logger.py
index a827642ba..09a2d1674 100644
--- a/src/pipecat/processors/logger.py
+++ b/src/pipecat/processors/logger.py
@@ -31,6 +31,8 @@ class FrameLogger(FrameProcessor):
self._ignored_frame_types = tuple(ignored_frame_types) if ignored_frame_types else None
async def process_frame(self, frame: Frame, direction: FrameDirection):
+ await super().process_frame(frame, direction)
+
if self._ignored_frame_types and not isinstance(frame, self._ignored_frame_types):
dir = "<" if direction is FrameDirection.UPSTREAM else ">"
msg = f"{dir} {self._prefix}: {frame}"
diff --git a/src/pipecat/processors/transcript_processor.py b/src/pipecat/processors/transcript_processor.py
index 5379e22e6..d31310f84 100644
--- a/src/pipecat/processors/transcript_processor.py
+++ b/src/pipecat/processors/transcript_processor.py
@@ -4,17 +4,23 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
-from typing import List
+from typing import List, Optional
+
+from loguru import logger
from pipecat.frames.frames import (
+ BotStoppedSpeakingFrame,
+ CancelFrame,
+ EndFrame,
Frame,
- OpenAILLMContextAssistantTimestampFrame,
+ StartInterruptionFrame,
TranscriptionFrame,
TranscriptionMessage,
TranscriptionUpdateFrame,
+ TTSTextFrame,
)
-from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.utils.time import time_now_iso8601
class BaseTranscriptProcessor(FrameProcessor):
@@ -64,89 +70,74 @@ class UserTranscriptProcessor(BaseTranscriptProcessor):
class AssistantTranscriptProcessor(BaseTranscriptProcessor):
- """Processes assistant LLM context frames into timestamped conversation messages."""
+ """Processes assistant TTS text frames into timestamped conversation messages.
+
+ This processor aggregates TTS text frames into complete utterances and emits them as
+ transcript messages. Utterances are completed when:
+ - The bot stops speaking (BotStoppedSpeakingFrame)
+ - The bot is interrupted (StartInterruptionFrame)
+ - The pipeline ends (EndFrame)
+
+ Attributes:
+ _current_text_parts: List of text fragments being aggregated for current utterance
+ _aggregation_start_time: Timestamp when the current utterance began
+ """
def __init__(self, **kwargs):
- """Initialize processor with empty message stores."""
+ """Initialize processor with aggregation state."""
super().__init__(**kwargs)
- self._pending_assistant_messages: List[TranscriptionMessage] = []
+ self._current_text_parts: List[str] = []
+ self._aggregation_start_time: Optional[str] | None = None
- def _extract_messages(self, messages: List[dict]) -> List[TranscriptionMessage]:
- """Extract assistant messages from the OpenAI standard message format.
+ async def _emit_aggregated_text(self):
+ """Emit aggregated text as a transcript message."""
+ if self._current_text_parts and self._aggregation_start_time:
+ content = " ".join(self._current_text_parts).strip()
+ if content:
+ logger.debug(f"Emitting aggregated assistant message: {content}")
+ message = TranscriptionMessage(
+ role="assistant",
+ content=content,
+ timestamp=self._aggregation_start_time,
+ )
+ await self._emit_update([message])
+ else:
+ logger.debug("No content to emit after stripping whitespace")
- Args:
- messages: List of messages in OpenAI format, which can be either:
- - Simple format: {"role": "user", "content": "Hello"}
- - Content list: {"role": "user", "content": [{"type": "text", "text": "Hello"}]}
-
- Returns:
- List[TranscriptionMessage]: Normalized conversation messages
- """
- result = []
- for msg in messages:
- if msg["role"] != "assistant":
- continue
-
- content = msg.get("content")
- if isinstance(content, str):
- if content:
- result.append(TranscriptionMessage(role="assistant", content=content))
- elif isinstance(content, list):
- text_parts = []
- for part in content:
- if isinstance(part, dict) and part.get("type") == "text":
- text_parts.append(part["text"])
-
- if text_parts:
- result.append(
- TranscriptionMessage(role="assistant", content=" ".join(text_parts))
- )
-
- return result
-
- def _find_new_messages(self, current: List[TranscriptionMessage]) -> List[TranscriptionMessage]:
- """Find unprocessed messages from current list.
-
- Args:
- current: List of current messages
-
- Returns:
- List[TranscriptionMessage]: New messages not yet processed
- """
- if not self._processed_messages:
- return current
-
- processed_len = len(self._processed_messages)
- if len(current) <= processed_len:
- return []
-
- return current[processed_len:]
+ # Reset aggregation state
+ self._current_text_parts = []
+ self._aggregation_start_time = None
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames into assistant conversation messages.
+ Handles different frame types:
+ - TTSTextFrame: Aggregates text for current utterance
+ - BotStoppedSpeakingFrame: Completes current utterance
+ - StartInterruptionFrame: Completes current utterance due to interruption
+ - EndFrame: Completes current utterance at pipeline end
+ - CancelFrame: Completes current utterance due to cancellation
+
Args:
frame: Input frame to process
direction: Frame processing direction
"""
await super().process_frame(frame, direction)
- if isinstance(frame, OpenAILLMContextFrame):
- standard_messages = []
- for msg in frame.context.messages:
- converted = frame.context.to_standard_messages(msg)
- standard_messages.extend(converted)
+ if isinstance(frame, TTSTextFrame):
+ # Start timestamp on first text part
+ if not self._aggregation_start_time:
+ self._aggregation_start_time = time_now_iso8601()
- current_messages = self._extract_messages(standard_messages)
- new_messages = self._find_new_messages(current_messages)
- self._pending_assistant_messages.extend(new_messages)
+ self._current_text_parts.append(frame.text)
- elif isinstance(frame, OpenAILLMContextAssistantTimestampFrame):
- if self._pending_assistant_messages:
- for msg in self._pending_assistant_messages:
- msg.timestamp = frame.timestamp
- await self._emit_update(self._pending_assistant_messages)
- self._pending_assistant_messages = []
+ elif isinstance(frame, (BotStoppedSpeakingFrame, StartInterruptionFrame, CancelFrame)):
+ # Emit accumulated text when bot finishes speaking or is interrupted
+ await self._emit_aggregated_text()
+
+ elif isinstance(frame, EndFrame):
+ # Emit any remaining text when pipeline ends
+ await self._emit_aggregated_text()
await self.push_frame(frame, direction)
@@ -170,8 +161,8 @@ class TranscriptProcessor:
llm,
tts,
transport.output(),
+ transcript.assistant_tts(), # Assistant transcripts
context_aggregator.assistant(),
- transcript.assistant(), # Assistant transcripts
]
)
diff --git a/src/pipecat/processors/user_idle_processor.py b/src/pipecat/processors/user_idle_processor.py
index 3a7202c80..e65bd437f 100644
--- a/src/pipecat/processors/user_idle_processor.py
+++ b/src/pipecat/processors/user_idle_processor.py
@@ -5,7 +5,8 @@
#
import asyncio
-from typing import Awaitable, Callable
+import inspect
+from typing import Awaitable, Callable, Union
from pipecat.frames.frames import (
BotSpeakingFrame,
@@ -25,11 +26,23 @@ class UserIdleProcessor(FrameProcessor):
or BotSpeaking).
Args:
- callback: Function to call when user is idle
+ callback: Function to call when user is idle. Can be either:
+ - Basic callback(processor) -> None
+ - Retry callback(processor, retry_count) -> bool
+ Return True to continue monitoring for idle events,
+ Return False to stop the idle monitoring task
timeout: Seconds to wait before considering user idle
**kwargs: Additional arguments passed to FrameProcessor
Example:
+ # Retry callback:
+ async def handle_idle(processor: "UserIdleProcessor", retry_count: int) -> bool:
+ if retry_count < 3:
+ await send_reminder("Are you still there?")
+ return True
+ return False
+
+ # Basic callback:
async def handle_idle(processor: "UserIdleProcessor") -> None:
await send_reminder("Are you still there?")
@@ -42,34 +55,68 @@ class UserIdleProcessor(FrameProcessor):
def __init__(
self,
*,
- callback: Callable[["UserIdleProcessor"], Awaitable[None]],
+ callback: Union[
+ Callable[["UserIdleProcessor"], Awaitable[None]], # Basic
+ Callable[["UserIdleProcessor", int], Awaitable[bool]], # Retry
+ ],
timeout: float,
**kwargs,
):
super().__init__(**kwargs)
- self._callback = callback
+ self._callback = self._wrap_callback(callback)
self._timeout = timeout
+ self._retry_count = 0
self._interrupted = False
self._conversation_started = False
self._idle_task = None
self._idle_event = asyncio.Event()
- def _create_idle_task(self):
- """Create the idle task if it hasn't been created yet."""
- if self._idle_task is None:
- self._idle_task = self.get_event_loop().create_task(self._idle_task_handler())
+ def _wrap_callback(
+ self,
+ callback: Union[
+ Callable[["UserIdleProcessor"], Awaitable[None]],
+ Callable[["UserIdleProcessor", int], Awaitable[bool]],
+ ],
+ ) -> Callable[["UserIdleProcessor", int], Awaitable[bool]]:
+ """Wraps callback to support both basic and retry signatures.
- async def _stop(self):
+ Args:
+ callback: The callback function to wrap.
+
+ Returns:
+ A wrapped callback that returns bool to indicate whether to continue monitoring.
+ """
+ sig = inspect.signature(callback)
+ param_count = len(sig.parameters)
+
+ async def wrapper(processor: "UserIdleProcessor", retry_count: int) -> bool:
+ if param_count == 1:
+ # Basic callback
+ await callback(processor) # type: ignore
+ return True
+ else:
+ # Retry callback
+ return await callback(processor, retry_count) # type: ignore
+
+ return wrapper
+
+ def _create_idle_task(self) -> None:
+ """Creates the idle task if it hasn't been created yet."""
+ if self._idle_task is None:
+ self._idle_task = self.create_task(self._idle_task_handler())
+
+ @property
+ def retry_count(self) -> int:
+ """Returns the current retry count."""
+ return self._retry_count
+
+ async def _stop(self) -> None:
"""Stops and cleans up the idle monitoring task."""
if self._idle_task is not None:
- self._idle_task.cancel()
- try:
- await self._idle_task
- except asyncio.CancelledError:
- pass # Expected when task is cancelled
+ await self.cancel_task(self._idle_task)
self._idle_task = None
- async def process_frame(self, frame: Frame, direction: FrameDirection):
+ async def process_frame(self, frame: Frame, direction: FrameDirection) -> None:
"""Processes incoming frames and manages idle monitoring state.
Args:
@@ -98,6 +145,7 @@ class UserIdleProcessor(FrameProcessor):
if self._conversation_started:
# We shouldn't call the idle callback if the user or the bot are speaking
if isinstance(frame, UserStartedSpeakingFrame):
+ self._retry_count = 0 # Reset retry count when user speaks
self._interrupted = True
self._idle_event.set()
elif isinstance(frame, UserStoppedSpeakingFrame):
@@ -106,23 +154,26 @@ class UserIdleProcessor(FrameProcessor):
elif isinstance(frame, BotSpeakingFrame):
self._idle_event.set()
- async def cleanup(self):
+ async def cleanup(self) -> None:
"""Cleans up resources when processor is shutting down."""
+ await super().cleanup()
if self._idle_task: # Only stop if task exists
await self._stop()
- async def _idle_task_handler(self):
+ async def _idle_task_handler(self) -> None:
"""Monitors for idle timeout and triggers callbacks.
- Runs in a loop until cancelled.
+ Runs in a loop until cancelled or callback indicates completion.
"""
while True:
try:
await asyncio.wait_for(self._idle_event.wait(), timeout=self._timeout)
except asyncio.TimeoutError:
if not self._interrupted:
- await self._callback(self)
- except asyncio.CancelledError:
- break
+ self._retry_count += 1
+ should_continue = await self._callback(self, self._retry_count)
+ if not should_continue:
+ await self._stop()
+ break
finally:
self._idle_event.clear()
diff --git a/src/pipecat/serializers/base_serializer.py b/src/pipecat/serializers/base_serializer.py
index 428649131..c70b1408e 100644
--- a/src/pipecat/serializers/base_serializer.py
+++ b/src/pipecat/serializers/base_serializer.py
@@ -7,7 +7,7 @@
from abc import ABC, abstractmethod
from enum import Enum
-from pipecat.frames.frames import Frame
+from pipecat.frames.frames import Frame, StartFrame
class FrameSerializerType(Enum):
@@ -21,10 +21,13 @@ class FrameSerializer(ABC):
def type(self) -> FrameSerializerType:
pass
- @abstractmethod
- def serialize(self, frame: Frame) -> str | bytes | None:
+ async def setup(self, frame: StartFrame):
pass
@abstractmethod
- def deserialize(self, data: str | bytes) -> Frame | None:
+ async def serialize(self, frame: Frame) -> str | bytes | None:
+ pass
+
+ @abstractmethod
+ async def deserialize(self, data: str | bytes) -> Frame | None:
pass
diff --git a/src/pipecat/serializers/livekit.py b/src/pipecat/serializers/livekit.py
index 760097ffe..d856a7a56 100644
--- a/src/pipecat/serializers/livekit.py
+++ b/src/pipecat/serializers/livekit.py
@@ -25,7 +25,7 @@ class LivekitFrameSerializer(FrameSerializer):
def type(self) -> FrameSerializerType:
return FrameSerializerType.BINARY
- def serialize(self, frame: Frame) -> str | bytes | None:
+ async def serialize(self, frame: Frame) -> str | bytes | None:
if not isinstance(frame, OutputAudioRawFrame):
return None
audio_frame = AudioFrame(
@@ -36,7 +36,7 @@ class LivekitFrameSerializer(FrameSerializer):
)
return pickle.dumps(audio_frame)
- def deserialize(self, data: str | bytes) -> Frame | None:
+ async def deserialize(self, data: str | bytes) -> Frame | None:
audio_frame: AudioFrame = pickle.loads(data)["frame"]
return InputAudioRawFrame(
audio=bytes(audio_frame.data),
diff --git a/src/pipecat/serializers/protobuf.py b/src/pipecat/serializers/protobuf.py
index f35a1e17a..125f2037f 100644
--- a/src/pipecat/serializers/protobuf.py
+++ b/src/pipecat/serializers/protobuf.py
@@ -41,7 +41,7 @@ class ProtobufFrameSerializer(FrameSerializer):
def type(self) -> FrameSerializerType:
return FrameSerializerType.BINARY
- def serialize(self, frame: Frame) -> str | bytes | None:
+ async def serialize(self, frame: Frame) -> str | bytes | None:
proto_frame = frame_protos.Frame()
if type(frame) not in self.SERIALIZABLE_TYPES:
logger.warning(f"Frame type {type(frame)} is not serializable")
@@ -57,26 +57,7 @@ class ProtobufFrameSerializer(FrameSerializer):
return proto_frame.SerializeToString()
- def deserialize(self, data: str | bytes) -> Frame | None:
- """Returns a Frame object from a Frame protobuf.
-
- Used to convert frames
- passed over the wire as protobufs to Frame objects used in pipelines
- and frame processors.
-
- >>> serializer = ProtobufFrameSerializer()
- >>> serializer.deserialize(
- ... serializer.serialize(OutputAudioFrame(data=b'1234567890')))
- InputAudioFrame(data=b'1234567890')
-
- >>> serializer.deserialize(
- ... serializer.serialize(TextFrame(text='hello world')))
- TextFrame(text='hello world')
-
- >>> serializer.deserialize(serializer.serialize(TranscriptionFrame(
- ... text="Hello there!", participantId="123", timestamp="2021-01-01")))
- TranscriptionFrame(text='Hello there!', participantId='123', timestamp='2021-01-01')
- """
+ async def deserialize(self, data: str | bytes) -> Frame | None:
proto = frame_protos.Frame.FromString(data)
which = proto.WhichOneof("frame")
if which not in self.DESERIALIZABLE_FIELDS:
@@ -93,11 +74,11 @@ class ProtobufFrameSerializer(FrameSerializer):
id = getattr(args, "id", None)
name = getattr(args, "name", None)
pts = getattr(args, "pts", None)
- if not id and "id" in args_dict:
+ if "id" in args_dict:
del args_dict["id"]
- if not name and "name" in args_dict:
+ if "name" in args_dict:
del args_dict["name"]
- if not pts and "pts" in args_dict:
+ if "pts" in args_dict:
del args_dict["pts"]
# Create the instance
@@ -105,10 +86,10 @@ class ProtobufFrameSerializer(FrameSerializer):
# Set special fields
if id:
- setattr(instance, "id", getattr(args, "id", None))
+ setattr(instance, "id", id)
if name:
- setattr(instance, "name", getattr(args, "name", None))
+ setattr(instance, "name", name)
if pts:
- setattr(instance, "pts", getattr(args, "pts", None))
+ setattr(instance, "pts", pts)
return instance
diff --git a/src/pipecat/serializers/telnyx.py b/src/pipecat/serializers/telnyx.py
new file mode 100644
index 000000000..9bff7535f
--- /dev/null
+++ b/src/pipecat/serializers/telnyx.py
@@ -0,0 +1,125 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import base64
+import json
+from typing import Optional
+
+from pydantic import BaseModel
+
+from pipecat.audio.utils import (
+ alaw_to_pcm,
+ create_default_resampler,
+ pcm_to_alaw,
+ pcm_to_ulaw,
+ ulaw_to_pcm,
+)
+from pipecat.frames.frames import (
+ AudioRawFrame,
+ Frame,
+ InputAudioRawFrame,
+ InputDTMFFrame,
+ KeypadEntry,
+ StartFrame,
+ StartInterruptionFrame,
+)
+from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializerType
+
+
+class TelnyxFrameSerializer(FrameSerializer):
+ class InputParams(BaseModel):
+ telnyx_sample_rate: Optional[int] = None
+ sample_rate: Optional[int] = None
+ inbound_encoding: str = "PCMU"
+ outbound_encoding: str = "PCMU"
+
+ def __init__(
+ self,
+ stream_id: str,
+ outbound_encoding: str,
+ inbound_encoding: str,
+ params: InputParams = InputParams(),
+ ):
+ self._stream_id = stream_id
+ params.outbound_encoding = outbound_encoding
+ params.inbound_encoding = inbound_encoding
+ self._params = params
+
+ self._resampler = create_default_resampler()
+
+ @property
+ def type(self) -> FrameSerializerType:
+ return FrameSerializerType.TEXT
+
+ async def setup(self, frame: StartFrame):
+ self._telnyx_sample_rate = self._params.telnyx_sample_rate or frame.audio_in_sample_rate
+ self._sample_rate = self._params.sample_rate or frame.audio_out_sample_rate
+
+ async def serialize(self, frame: Frame) -> str | bytes | None:
+ if isinstance(frame, AudioRawFrame):
+ data = frame.audio
+
+ if self._params.inbound_encoding == "PCMU":
+ serialized_data = await pcm_to_ulaw(
+ data, frame.sample_rate, self._telnyx_sample_rate, self._resampler
+ )
+ elif self._params.inbound_encoding == "PCMA":
+ serialized_data = await pcm_to_alaw(
+ data, frame.sample_rate, self._telnyx_sample_rate, self._resampler
+ )
+ else:
+ raise ValueError(f"Unsupported encoding: {self._params.inbound_encoding}")
+
+ payload = base64.b64encode(serialized_data).decode("utf-8")
+ answer = {
+ "event": "media",
+ "media": {"payload": payload},
+ }
+
+ return json.dumps(answer)
+
+ if isinstance(frame, StartInterruptionFrame):
+ answer = {"event": "clear"}
+ return json.dumps(answer)
+
+ async def deserialize(self, data: str | bytes) -> Frame | None:
+ message = json.loads(data)
+
+ if message["event"] == "media":
+ payload_base64 = message["media"]["payload"]
+ payload = base64.b64decode(payload_base64)
+
+ if self._params.outbound_encoding == "PCMU":
+ deserialized_data = await ulaw_to_pcm(
+ payload,
+ self._telnyx_sample_rate,
+ self._sample_rate,
+ self._resampler,
+ )
+ elif self._params.outbound_encoding == "PCMA":
+ deserialized_data = await alaw_to_pcm(
+ payload,
+ self._telnyx_sample_rate,
+ self._sample_rate,
+ self._resampler,
+ )
+ else:
+ raise ValueError(f"Unsupported encoding: {self._params.outbound_encoding}")
+
+ audio_frame = InputAudioRawFrame(
+ audio=deserialized_data, num_channels=1, sample_rate=self._sample_rate
+ )
+ return audio_frame
+ elif message["event"] == "dtmf":
+ digit = message.get("dtmf", {}).get("digit")
+
+ try:
+ return InputDTMFFrame(KeypadEntry(digit))
+ except ValueError as e:
+ # Handle case where string doesn't match any enum value
+ return None
+ else:
+ return None
diff --git a/src/pipecat/serializers/twilio.py b/src/pipecat/serializers/twilio.py
index 298e8a6cb..0ca37c221 100644
--- a/src/pipecat/serializers/twilio.py
+++ b/src/pipecat/serializers/twilio.py
@@ -6,39 +6,57 @@
import base64
import json
+from typing import Optional
from pydantic import BaseModel
-from pipecat.audio.utils import pcm_to_ulaw, ulaw_to_pcm
+from pipecat.audio.utils import create_default_resampler, pcm_to_ulaw, ulaw_to_pcm
from pipecat.frames.frames import (
AudioRawFrame,
Frame,
InputAudioRawFrame,
InputDTMFFrame,
KeypadEntry,
+ StartFrame,
StartInterruptionFrame,
+ TransportMessageFrame,
+ TransportMessageUrgentFrame,
)
from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializerType
class TwilioFrameSerializer(FrameSerializer):
class InputParams(BaseModel):
- twilio_sample_rate: int = 8000
- sample_rate: int = 16000
+ twilio_sample_rate: Optional[int] = None
+ sample_rate: Optional[int] = None
def __init__(self, stream_sid: str, params: InputParams = InputParams()):
self._stream_sid = stream_sid
self._params = params
+ self._twilio_sample_rate = 0
+ self._sample_rate = 0
+
+ self._resampler = create_default_resampler()
+
@property
def type(self) -> FrameSerializerType:
return FrameSerializerType.TEXT
- def serialize(self, frame: Frame) -> str | bytes | None:
- if isinstance(frame, AudioRawFrame):
+ async def setup(self, frame: StartFrame):
+ self._twilio_sample_rate = self._params.twilio_sample_rate or frame.audio_in_sample_rate
+ self._sample_rate = self._params.sample_rate or frame.audio_out_sample_rate
+
+ async def serialize(self, frame: Frame) -> str | bytes | None:
+ if isinstance(frame, StartInterruptionFrame):
+ answer = {"event": "clear", "streamSid": self._stream_sid}
+ return json.dumps(answer)
+ elif isinstance(frame, AudioRawFrame):
data = frame.audio
- serialized_data = pcm_to_ulaw(data, frame.sample_rate, self._params.twilio_sample_rate)
+ serialized_data = await pcm_to_ulaw(
+ data, frame.sample_rate, self._twilio_sample_rate, self._resampler
+ )
payload = base64.b64encode(serialized_data).decode("utf-8")
answer = {
"event": "media",
@@ -47,23 +65,21 @@ class TwilioFrameSerializer(FrameSerializer):
}
return json.dumps(answer)
+ elif isinstance(frame, (TransportMessageFrame, TransportMessageUrgentFrame)):
+ return json.dumps(frame.message)
- if isinstance(frame, StartInterruptionFrame):
- answer = {"event": "clear", "streamSid": self._stream_sid}
- return json.dumps(answer)
-
- def deserialize(self, data: str | bytes) -> Frame | None:
+ async def deserialize(self, data: str | bytes) -> Frame | None:
message = json.loads(data)
if message["event"] == "media":
payload_base64 = message["media"]["payload"]
payload = base64.b64decode(payload_base64)
- deserialized_data = ulaw_to_pcm(
- payload, self._params.twilio_sample_rate, self._params.sample_rate
+ deserialized_data = await ulaw_to_pcm(
+ payload, self._twilio_sample_rate, self._sample_rate, self._resampler
)
audio_frame = InputAudioRawFrame(
- audio=deserialized_data, num_channels=1, sample_rate=self._params.sample_rate
+ audio=deserialized_data, num_channels=1, sample_rate=self._sample_rate
)
return audio_frame
elif message["event"] == "dtmf":
diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py
index bfd155d38..1bbdb4340 100644
--- a/src/pipecat/services/ai_services.py
+++ b/src/pipecat/services/ai_services.py
@@ -8,7 +8,7 @@ import asyncio
import io
import wave
from abc import abstractmethod
-from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple
+from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Tuple
from loguru import logger
@@ -69,7 +69,7 @@ class AIService(FrameProcessor):
async def cancel(self, frame: CancelFrame):
pass
- async def _update_settings(self, settings: Dict[str, Any]):
+ async def _update_settings(self, settings: Mapping[str, Any]):
from pipecat.services.openai_realtime_beta.events import (
SessionProperties,
)
@@ -208,8 +208,12 @@ class TTSService(AIService):
push_stop_frames: bool = False,
# if push_stop_frames is True, wait for this idle period before pushing TTSStoppedFrame
stop_frame_timeout_s: float = 1.0,
+ # if True, TTSService will push silence audio frames after TTSStoppedFrame
+ push_silence_after_stop: bool = False,
+ # if push_silence_after_stop is True, send this amount of audio silence
+ silence_time_s: float = 2.0,
# TTS output sample rate
- sample_rate: int = 24000,
+ sample_rate: Optional[int] = None,
text_filter: Optional[BaseTextFilter] = None,
**kwargs,
):
@@ -218,7 +222,10 @@ class TTSService(AIService):
self._push_text_frames: bool = push_text_frames
self._push_stop_frames: bool = push_stop_frames
self._stop_frame_timeout_s: float = stop_frame_timeout_s
- self._sample_rate: int = sample_rate
+ self._push_silence_after_stop: bool = push_silence_after_stop
+ self._silence_time_s: float = silence_time_s
+ self._init_sample_rate = sample_rate
+ self._sample_rate = 0
self._voice_id: str = ""
self._settings: Dict[str, Any] = {}
self._text_filter: Optional[BaseTextFilter] = text_filter
@@ -242,34 +249,36 @@ class TTSService(AIService):
async def flush_audio(self):
pass
- def language_to_service_language(self, language: Language) -> str | None:
- return Language(language)
-
# Converts the text to audio.
@abstractmethod
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
pass
+ def language_to_service_language(self, language: Language) -> str | None:
+ return Language(language)
+
+ async def update_setting(self, key: str, value: Any):
+ pass
+
async def start(self, frame: StartFrame):
await super().start(frame)
+ self._sample_rate = self._init_sample_rate or frame.audio_out_sample_rate
if self._push_stop_frames:
- self._stop_frame_task = self.get_event_loop().create_task(self._stop_frame_handler())
+ self._stop_frame_task = self.create_task(self._stop_frame_handler())
async def stop(self, frame: EndFrame):
await super().stop(frame)
if self._stop_frame_task:
- self._stop_frame_task.cancel()
- await self._stop_frame_task
+ await self.cancel_task(self._stop_frame_task)
self._stop_frame_task = None
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
if self._stop_frame_task:
- self._stop_frame_task.cancel()
- await self._stop_frame_task
+ await self.cancel_task(self._stop_frame_task)
self._stop_frame_task = None
- async def _update_settings(self, settings: Dict[str, Any]):
+ async def _update_settings(self, settings: Mapping[str, Any]):
for key, value in settings.items():
if key in self._settings:
logger.info(f"Updating TTS setting {key} to: [{value}]")
@@ -316,6 +325,16 @@ class TTSService(AIService):
await self.push_frame(frame, direction)
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
+ if self._push_silence_after_stop and isinstance(frame, TTSStoppedFrame):
+ silence_num_bytes = int(self._silence_time_s * self.sample_rate * 2) # 16-bit
+ await self.push_frame(
+ TTSAudioRawFrame(
+ audio=b"\x00" * silence_num_bytes,
+ sample_rate=self.sample_rate,
+ num_channels=1,
+ )
+ )
+
await super().push_frame(frame, direction)
if self._push_stop_frames and (
@@ -364,23 +383,20 @@ class TTSService(AIService):
await self.push_frame(TTSTextFrame(text))
async def _stop_frame_handler(self):
- try:
- has_started = False
- while True:
- try:
- frame = await asyncio.wait_for(
- self._stop_frame_queue.get(), self._stop_frame_timeout_s
- )
- if isinstance(frame, TTSStartedFrame):
- has_started = True
- elif isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)):
- has_started = False
- except asyncio.TimeoutError:
- if has_started:
- await self.push_frame(TTSStoppedFrame())
- has_started = False
- except asyncio.CancelledError:
- pass
+ has_started = False
+ while True:
+ try:
+ frame = await asyncio.wait_for(
+ self._stop_frame_queue.get(), self._stop_frame_timeout_s
+ )
+ if isinstance(frame, TTSStartedFrame):
+ has_started = True
+ elif isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)):
+ has_started = False
+ except asyncio.TimeoutError:
+ if has_started:
+ await self.push_frame(TTSStoppedFrame())
+ has_started = False
class WordTTSService(TTSService):
@@ -388,7 +404,7 @@ class WordTTSService(TTSService):
super().__init__(**kwargs)
self._initial_word_timestamp = -1
self._words_queue = asyncio.Queue()
- self._words_task = self.get_event_loop().create_task(self._words_task_handler())
+ self._words_task = None
def start_word_timestamps(self):
if self._initial_word_timestamp == -1:
@@ -401,6 +417,10 @@ class WordTTSService(TTSService):
for word, timestamp in word_times:
await self._words_queue.put((word, seconds_to_nanoseconds(timestamp)))
+ async def start(self, frame: StartFrame):
+ await super().start(frame)
+ await self._create_words_task()
+
async def stop(self, frame: EndFrame):
await super().stop(frame)
await self._stop_words_task()
@@ -419,45 +439,50 @@ class WordTTSService(TTSService):
await super()._handle_interruption(frame, direction)
self.reset_word_timestamps()
+ async def _create_words_task(self):
+ self._words_task = self.create_task(self._words_task_handler())
+
async def _stop_words_task(self):
if self._words_task:
- self._words_task.cancel()
- await self._words_task
+ await self.cancel_task(self._words_task)
self._words_task = None
async def _words_task_handler(self):
last_pts = 0
while True:
- try:
- (word, timestamp) = await self._words_queue.get()
- if word == "Reset" and timestamp == 0:
- self.reset_word_timestamps()
- frame = None
- elif word == "LLMFullResponseEndFrame" and timestamp == 0:
- frame = LLMFullResponseEndFrame()
- frame.pts = last_pts
- elif word == "TTSStoppedFrame" and timestamp == 0:
- frame = TTSStoppedFrame()
- frame.pts = last_pts
- else:
- frame = TTSTextFrame(word)
- frame.pts = self._initial_word_timestamp + timestamp
- if frame:
- last_pts = frame.pts
- await self.push_frame(frame)
- self._words_queue.task_done()
- except asyncio.CancelledError:
- break
- except Exception as e:
- logger.exception(f"{self} exception: {e}")
+ (word, timestamp) = await self._words_queue.get()
+ if word == "Reset" and timestamp == 0:
+ self.reset_word_timestamps()
+ frame = None
+ elif word == "LLMFullResponseEndFrame" and timestamp == 0:
+ frame = LLMFullResponseEndFrame()
+ frame.pts = last_pts
+ elif word == "TTSStoppedFrame" and timestamp == 0:
+ frame = TTSStoppedFrame()
+ frame.pts = last_pts
+ else:
+ frame = TTSTextFrame(word)
+ frame.pts = self._initial_word_timestamp + timestamp
+ if frame:
+ last_pts = frame.pts
+ await self.push_frame(frame)
+ self._words_queue.task_done()
class STTService(AIService):
"""STTService is a base class for speech-to-text services."""
- def __init__(self, audio_passthrough=False, **kwargs):
+ def __init__(
+ self,
+ audio_passthrough=False,
+ # STT input sample rate
+ sample_rate: Optional[int] = None,
+ **kwargs,
+ ):
super().__init__(**kwargs)
self._audio_passthrough = audio_passthrough
+ self._init_sample_rate = sample_rate
+ self._sample_rate = 0
self._settings: Dict[str, Any] = {}
self._muted: bool = False
@@ -466,6 +491,10 @@ class STTService(AIService):
"""Returns whether the STT service is currently muted."""
return self._muted
+ @property
+ def sample_rate(self) -> int:
+ return self._sample_rate
+
@abstractmethod
async def set_model(self, model: str):
self.set_model_name(model)
@@ -479,7 +508,11 @@ class STTService(AIService):
"""Returns transcript as a string"""
pass
- async def _update_settings(self, settings: Dict[str, Any]):
+ async def start(self, frame: StartFrame):
+ await super().start(frame)
+ self._sample_rate = self._init_sample_rate or frame.audio_in_sample_rate
+
+ async def _update_settings(self, settings: Mapping[str, Any]):
logger.info(f"Updating STT settings: {self._settings}")
for key, value in settings.items():
if key in self._settings:
@@ -528,17 +561,15 @@ class SegmentedSTTService(STTService):
min_volume: float = 0.6,
max_silence_secs: float = 0.3,
max_buffer_secs: float = 1.5,
- sample_rate: int = 24000,
- num_channels: int = 1,
+ sample_rate: Optional[int] = None,
**kwargs,
):
- super().__init__(**kwargs)
+ super().__init__(sample_rate=sample_rate, **kwargs)
self._min_volume = min_volume
self._max_silence_secs = max_silence_secs
self._max_buffer_secs = max_buffer_secs
- self._sample_rate = sample_rate
- self._num_channels = num_channels
- (self._content, self._wave) = self._new_wave()
+ self._content = None
+ self._wave = None
self._silence_num_frames = 0
# Volume exponential smoothing
self._smoothing_factor = 0.2
@@ -557,8 +588,8 @@ class SegmentedSTTService(STTService):
# If buffer is not empty and we have enough data or there's been a long
# silence, transcribe the audio gathered so far.
- silence_secs = self._silence_num_frames / self._sample_rate
- buffer_secs = self._wave.getnframes() / self._sample_rate
+ silence_secs = self._silence_num_frames / self.sample_rate
+ buffer_secs = self._wave.getnframes() / self.sample_rate
if self._content.tell() > 0 and (
buffer_secs > self._max_buffer_secs or silence_secs > self._max_silence_secs
):
@@ -568,18 +599,24 @@ class SegmentedSTTService(STTService):
await self.process_generator(self.run_stt(self._content.read()))
(self._content, self._wave) = self._new_wave()
+ async def start(self, frame: StartFrame):
+ await super().start(frame)
+ (self._content, self._wave) = self._new_wave()
+
async def stop(self, frame: EndFrame):
+ await super().stop(frame)
self._wave.close()
async def cancel(self, frame: CancelFrame):
+ await super().cancel(frame)
self._wave.close()
def _new_wave(self):
content = io.BytesIO()
ww = wave.open(content, "wb")
ww.setsampwidth(2)
- ww.setnchannels(self._num_channels)
- ww.setframerate(self._sample_rate)
+ ww.setnchannels(1)
+ ww.setframerate(self.sample_rate)
return (content, ww)
def _get_smoothed_volume(self, frame: AudioRawFrame) -> float:
diff --git a/src/pipecat/services/assemblyai.py b/src/pipecat/services/assemblyai.py
index 78783f8b4..7abfcc5c6 100644
--- a/src/pipecat/services/assemblyai.py
+++ b/src/pipecat/services/assemblyai.py
@@ -5,7 +5,7 @@
#
import asyncio
-from typing import AsyncGenerator
+from typing import AsyncGenerator, Optional
from loguru import logger
@@ -38,20 +38,17 @@ class AssemblyAISTTService(STTService):
self,
*,
api_key: str,
- sample_rate: int = 16000,
+ sample_rate: Optional[int] = None,
encoding: AudioEncoding = AudioEncoding("pcm_s16le"),
language=Language.EN, # Only English is supported for Realtime
**kwargs,
):
- super().__init__(**kwargs)
+ super().__init__(sample_rate=sample_rate, **kwargs)
aai.settings.api_key = api_key
self._transcriber: aai.RealtimeTranscriber | None = None
- # Store reference to the main event loop for use in callback functions
- self._loop = asyncio.get_event_loop()
self._settings = {
- "sample_rate": sample_rate,
"encoding": encoding,
"language": language,
}
@@ -121,7 +118,7 @@ class AssemblyAISTTService(STTService):
# Schedule the coroutine to run in the main event loop
# This is necessary because this callback runs in a different thread
- asyncio.run_coroutine_threadsafe(self.push_frame(frame), self._loop)
+ asyncio.run_coroutine_threadsafe(self.push_frame(frame), self.get_event_loop())
def on_error(error: aai.RealtimeError):
"""Callback for handling errors from AssemblyAI.
@@ -131,14 +128,16 @@ class AssemblyAISTTService(STTService):
"""
logger.error(f"{self}: An error occurred: {error}")
# Schedule the coroutine to run in the main event loop
- asyncio.run_coroutine_threadsafe(self.push_frame(ErrorFrame(str(error))), self._loop)
+ asyncio.run_coroutine_threadsafe(
+ self.push_frame(ErrorFrame(str(error))), self.get_event_loop()
+ )
def on_close():
"""Callback for when the connection to AssemblyAI is closed."""
logger.info(f"{self}: Disconnected from AssemblyAI")
self._transcriber = aai.RealtimeTranscriber(
- sample_rate=self._settings["sample_rate"],
+ sample_rate=self.sample_rate,
encoding=self._settings["encoding"],
on_data=on_data,
on_error=on_error,
diff --git a/src/pipecat/services/aws.py b/src/pipecat/services/aws.py
index 3061e13a1..fdbd73681 100644
--- a/src/pipecat/services/aws.py
+++ b/src/pipecat/services/aws.py
@@ -10,7 +10,7 @@ from typing import AsyncGenerator, Optional
from loguru import logger
from pydantic import BaseModel
-from pipecat.audio.utils import resample_audio
+from pipecat.audio.utils import create_default_resampler
from pipecat.frames.frames import (
ErrorFrame,
Frame,
@@ -124,7 +124,7 @@ class PollyTTSService(TTSService):
aws_session_token: Optional[str] = None,
region: Optional[str] = None,
voice_id: str = "Joanna",
- sample_rate: int = 24000,
+ sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
**kwargs,
):
@@ -138,7 +138,6 @@ class PollyTTSService(TTSService):
region_name=region,
)
self._settings = {
- "sample_rate": sample_rate,
"engine": params.engine,
"language": self.language_to_service_language(params.language)
if params.language
@@ -148,6 +147,8 @@ class PollyTTSService(TTSService):
"volume": params.volume,
}
+ self._resampler = create_default_resampler()
+
self.set_voice(voice_id)
def can_generate_metrics(self) -> bool:
@@ -193,8 +194,7 @@ class PollyTTSService(TTSService):
response = self._polly_client.synthesize_speech(**args)
if "AudioStream" in response:
audio_data = response["AudioStream"].read()
- resampled = resample_audio(audio_data, 16000, self._settings["sample_rate"])
- return resampled
+ return audio_data
return None
logger.debug(f"Generating TTS: [{text}]")
@@ -225,6 +225,8 @@ class PollyTTSService(TTSService):
yield None
return
+ audio_data = await self._resampler.resample(audio_data, 16000, self.sample_rate)
+
await self.start_tts_usage_metrics(text)
yield TTSStartedFrame()
@@ -234,7 +236,7 @@ class PollyTTSService(TTSService):
chunk = audio_data[i : i + chunk_size]
if len(chunk) > 0:
await self.stop_ttfb_metrics()
- frame = TTSAudioRawFrame(chunk, self._settings["sample_rate"], 1)
+ frame = TTSAudioRawFrame(chunk, self.sample_rate, 1)
yield frame
yield TTSStoppedFrame()
diff --git a/src/pipecat/services/azure.py b/src/pipecat/services/azure.py
index c7a0ce547..0781c04b9 100644
--- a/src/pipecat/services/azure.py
+++ b/src/pipecat/services/azure.py
@@ -450,14 +450,13 @@ class AzureBaseTTSService(TTSService):
api_key: str,
region: str,
voice="en-US-SaraNeural",
- sample_rate: int = 24000,
+ sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
self._settings = {
- "sample_rate": sample_rate,
"emphasis": params.emphasis,
"language": self.language_to_service_language(params.language)
if params.language
@@ -530,25 +529,32 @@ class AzureBaseTTSService(TTSService):
class AzureTTSService(AzureBaseTTSService):
def __init__(self, **kwargs):
super().__init__(**kwargs)
+ self._speech_config = None
+ self._speech_synthesizer = None
+ self._audio_queue = asyncio.Queue()
- speech_config = SpeechConfig(
+ async def start(self, frame: StartFrame):
+ await super().start(frame)
+ # Now self.sample_rate is properly initialized
+ self._speech_config = SpeechConfig(
subscription=self._api_key,
region=self._region,
speech_recognition_language=self._settings["language"],
)
- speech_config.set_speech_synthesis_output_format(
- sample_rate_to_output_format(self._settings["sample_rate"])
+ self._speech_config.set_speech_synthesis_output_format(
+ sample_rate_to_output_format(self.sample_rate)
)
- speech_config.set_service_property(
+ self._speech_config.set_service_property(
"synthesizer.synthesis.connection.synthesisConnectionImpl",
"websocket",
ServicePropertyChannel.UriQueryParameter,
)
- self._speech_synthesizer = SpeechSynthesizer(speech_config=speech_config, audio_config=None)
+ self._speech_synthesizer = SpeechSynthesizer(
+ speech_config=self._speech_config, audio_config=None
+ )
# Set up event handlers
- self._audio_queue = asyncio.Queue()
self._speech_synthesizer.synthesizing.connect(self._handle_synthesizing)
self._speech_synthesizer.synthesis_completed.connect(self._handle_completed)
self._speech_synthesizer.synthesis_canceled.connect(self._handle_canceled)
@@ -591,7 +597,7 @@ class AzureTTSService(AzureBaseTTSService):
yield TTSAudioRawFrame(
audio=chunk,
- sample_rate=self._settings["sample_rate"],
+ sample_rate=self.sample_rate,
num_channels=1,
)
@@ -605,17 +611,22 @@ class AzureTTSService(AzureBaseTTSService):
class AzureHttpTTSService(AzureBaseTTSService):
def __init__(self, **kwargs):
super().__init__(**kwargs)
+ self._speech_config = None
+ self._speech_synthesizer = None
- speech_config = SpeechConfig(
+ async def start(self, frame: StartFrame):
+ await super().start(frame)
+ self._speech_config = SpeechConfig(
subscription=self._api_key,
region=self._region,
speech_recognition_language=self._settings["language"],
)
- speech_config.set_speech_synthesis_output_format(
- sample_rate_to_output_format(self._settings["sample_rate"])
+ self._speech_config.set_speech_synthesis_output_format(
+ sample_rate_to_output_format(self.sample_rate)
+ )
+ self._speech_synthesizer = SpeechSynthesizer(
+ speech_config=self._speech_config, audio_config=None
)
-
- self._speech_synthesizer = SpeechSynthesizer(speech_config=speech_config, audio_config=None)
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
@@ -633,7 +644,7 @@ class AzureHttpTTSService(AzureBaseTTSService):
# Azure always sends a 44-byte header. Strip it off.
yield TTSAudioRawFrame(
audio=result.audio_data[44:],
- sample_rate=self._settings["sample_rate"],
+ sample_rate=self.sample_rate,
num_channels=1,
)
yield TTSStoppedFrame()
@@ -650,24 +661,14 @@ class AzureSTTService(STTService):
*,
api_key: str,
region: str,
- language=Language.EN_US,
- sample_rate=24000,
- channels=1,
+ language: Language = Language.EN_US,
+ sample_rate: Optional[int] = None,
**kwargs,
):
- super().__init__(**kwargs)
+ super().__init__(sample_rate=sample_rate, **kwargs)
- speech_config = SpeechConfig(subscription=api_key, region=region)
- speech_config.speech_recognition_language = language
-
- stream_format = AudioStreamFormat(samples_per_second=sample_rate, channels=channels)
- self._audio_stream = PushAudioInputStream(stream_format)
-
- audio_config = AudioConfig(stream=self._audio_stream)
- self._speech_recognizer = SpeechRecognizer(
- speech_config=speech_config, audio_config=audio_config
- )
- self._speech_recognizer.recognized.connect(self._on_handle_recognized)
+ self._speech_config = SpeechConfig(subscription=api_key, region=region)
+ self._speech_config.speech_recognition_language = language
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
await self.start_processing_metrics()
@@ -677,6 +678,16 @@ class AzureSTTService(STTService):
async def start(self, frame: StartFrame):
await super().start(frame)
+
+ stream_format = AudioStreamFormat(samples_per_second=self.sample_rate, channels=1)
+ self._audio_stream = PushAudioInputStream(stream_format)
+
+ audio_config = AudioConfig(stream=self._audio_stream)
+
+ self._speech_recognizer = SpeechRecognizer(
+ speech_config=self._speech_config, audio_config=audio_config
+ )
+ self._speech_recognizer.recognized.connect(self._on_handle_recognized)
self._speech_recognizer.start_continuous_recognition_async()
async def stop(self, frame: EndFrame):
diff --git a/src/pipecat/services/canonical.py b/src/pipecat/services/canonical.py
index 37077a955..31bd6e89a 100644
--- a/src/pipecat/services/canonical.py
+++ b/src/pipecat/services/canonical.py
@@ -18,6 +18,7 @@ from pipecat.frames.frames import CancelFrame, EndFrame, Frame
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor
from pipecat.processors.frame_processor import FrameDirection
+from pipecat.services.ai_services import AIService
try:
import aiofiles
@@ -116,7 +117,6 @@ class CanonicalMetricsService(AIService):
try:
await self._multipart_upload(filename)
await aiofiles.os.remove(filename)
- audio_buffer_processor.reset_audio_buffers()
except FileNotFoundError:
pass
except Exception as e:
diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py
index a91c06176..525071271 100644
--- a/src/pipecat/services/cartesia.py
+++ b/src/pipecat/services/cartesia.py
@@ -88,8 +88,8 @@ class CartesiaTTSService(WordTTSService, WebsocketService):
voice_id: str,
cartesia_version: str = "2024-06-10",
url: str = "wss://api.cartesia.ai/tts/websocket",
- model: str = "sonic-english",
- sample_rate: int = 24000,
+ model: str = "sonic",
+ sample_rate: Optional[int] = None,
encoding: str = "pcm_s16le",
container: str = "raw",
params: InputParams = InputParams(),
@@ -121,7 +121,7 @@ class CartesiaTTSService(WordTTSService, WebsocketService):
"output_format": {
"container": container,
"encoding": encoding,
- "sample_rate": sample_rate,
+ "sample_rate": 0,
},
"language": self.language_to_service_language(params.language)
if params.language
@@ -174,6 +174,7 @@ class CartesiaTTSService(WordTTSService, WebsocketService):
async def start(self, frame: StartFrame):
await super().start(frame)
+ self._settings["output_format"]["sample_rate"] = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -187,16 +188,13 @@ class CartesiaTTSService(WordTTSService, WebsocketService):
async def _connect(self):
await self._connect_websocket()
- self._receive_task = self.get_event_loop().create_task(
- self._receive_task_handler(self.push_error)
- )
+ self._receive_task = self.create_task(self._receive_task_handler(self.push_error))
async def _disconnect(self):
await self._disconnect_websocket()
if self._receive_task:
- self._receive_task.cancel()
- await self._receive_task
+ await self.cancel_task(self._receive_task)
self._receive_task = None
async def _connect_websocket(self):
@@ -265,7 +263,7 @@ class CartesiaTTSService(WordTTSService, WebsocketService):
self.start_word_timestamps()
frame = TTSAudioRawFrame(
audio=base64.b64decode(msg["data"]),
- sample_rate=self._settings["output_format"]["sample_rate"],
+ sample_rate=self.sample_rate,
num_channels=1,
)
await self.push_frame(frame)
@@ -329,9 +327,9 @@ class CartesiaHttpTTSService(TTSService):
*,
api_key: str,
voice_id: str,
- model: str = "sonic-english",
+ model: str = "sonic",
base_url: str = "https://api.cartesia.ai",
- sample_rate: int = 24000,
+ sample_rate: Optional[int] = None,
encoding: str = "pcm_s16le",
container: str = "raw",
params: InputParams = InputParams(),
@@ -344,7 +342,7 @@ class CartesiaHttpTTSService(TTSService):
"output_format": {
"container": container,
"encoding": encoding,
- "sample_rate": sample_rate,
+ "sample_rate": 0,
},
"language": self.language_to_service_language(params.language)
if params.language
@@ -363,6 +361,10 @@ class CartesiaHttpTTSService(TTSService):
def language_to_service_language(self, language: Language) -> str | None:
return language_to_cartesia_language(language)
+ async def start(self, frame: StartFrame):
+ await super().start(frame)
+ self._settings["output_format"]["sample_rate"] = self.sample_rate
+
async def stop(self, frame: EndFrame):
await super().stop(frame)
await self._client.close()
@@ -397,9 +399,7 @@ class CartesiaHttpTTSService(TTSService):
)
frame = TTSAudioRawFrame(
- audio=output["audio"],
- sample_rate=self._settings["output_format"]["sample_rate"],
- num_channels=1,
+ audio=output["audio"], sample_rate=self.sample_rate, num_channels=1
)
yield frame
except Exception as e:
diff --git a/src/pipecat/services/deepgram.py b/src/pipecat/services/deepgram.py
index a5d36370f..b9d39fa63 100644
--- a/src/pipecat/services/deepgram.py
+++ b/src/pipecat/services/deepgram.py
@@ -5,7 +5,7 @@
#
import asyncio
-from typing import AsyncGenerator
+from typing import AsyncGenerator, Optional
from loguru import logger
@@ -53,14 +53,13 @@ class DeepgramTTSService(TTSService):
*,
api_key: str,
voice: str = "aura-helios-en",
- sample_rate: int = 24000,
+ sample_rate: Optional[int] = None,
encoding: str = "linear16",
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
self._settings = {
- "sample_rate": sample_rate,
"encoding": encoding,
}
self.set_voice(voice)
@@ -75,7 +74,7 @@ class DeepgramTTSService(TTSService):
options = SpeakOptions(
model=self._voice_id,
encoding=self._settings["encoding"],
- sample_rate=self._settings["sample_rate"],
+ sample_rate=self.sample_rate,
container="none",
)
@@ -103,9 +102,7 @@ class DeepgramTTSService(TTSService):
chunk = audio_buffer.read(chunk_size)
if not chunk:
break
- frame = TTSAudioRawFrame(
- audio=chunk, sample_rate=self._settings["sample_rate"], num_channels=1
- )
+ frame = TTSAudioRawFrame(audio=chunk, sample_rate=self.sample_rate, num_channels=1)
yield frame
yield TTSStoppedFrame()
@@ -121,15 +118,16 @@ class DeepgramSTTService(STTService):
*,
api_key: str,
url: str = "",
- live_options: LiveOptions = None,
+ sample_rate: Optional[int] = None,
+ live_options: Optional[LiveOptions] = None,
**kwargs,
):
- super().__init__(**kwargs)
+ super().__init__(sample_rate=sample_rate, **kwargs)
+
default_options = LiveOptions(
encoding="linear16",
language=Language.EN,
model="nova-2-general",
- sample_rate=16000,
channels=1,
interim_results=True,
smart_format=True,
@@ -187,6 +185,7 @@ class DeepgramSTTService(STTService):
async def start(self, frame: StartFrame):
await super().start(frame)
+ self._settings["sample_rate"] = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py
index d80a7e9c4..13faa78a5 100644
--- a/src/pipecat/services/elevenlabs.py
+++ b/src/pipecat/services/elevenlabs.py
@@ -44,10 +44,14 @@ except ModuleNotFoundError as e:
ElevenLabsOutputFormat = Literal["pcm_16000", "pcm_22050", "pcm_24000", "pcm_44100"]
+# Models that support language codes
+# The following models are excluded as they don't support language codes:
+# - eleven_flash_v2
+# - eleven_turbo_v2
+# - eleven_multilingual_v2
ELEVENLABS_MULTILINGUAL_MODELS = {
- "eleven_turbo_v2_5",
- "eleven_multilingual_v2",
"eleven_flash_v2_5",
+ "eleven_turbo_v2_5",
}
@@ -100,17 +104,20 @@ def language_to_elevenlabs_language(language: Language) -> str | None:
return result
-def sample_rate_from_output_format(output_format: str) -> int:
- match output_format:
- case "pcm_16000":
- return 16000
- case "pcm_22050":
- return 22050
- case "pcm_24000":
- return 24000
- case "pcm_44100":
- return 44100
- return 16000
+def output_format_from_sample_rate(sample_rate: int) -> str:
+ match sample_rate:
+ case 16000:
+ return "pcm_16000"
+ case 22050:
+ return "pcm_22050"
+ case 24000:
+ return "pcm_24000"
+ case 44100:
+ return "pcm_44100"
+ logger.warning(
+ f"ElevenLabsTTSService: No output format available for {sample_rate} sample rate"
+ )
+ return "pcm_16000"
def calculate_word_times(
@@ -136,7 +143,7 @@ def calculate_word_times(
class ElevenLabsTTSService(WordTTSService, WebsocketService):
class InputParams(BaseModel):
- language: Optional[Language] = Language.EN
+ language: Optional[Language] = None
optimize_streaming_latency: Optional[str] = None
stability: Optional[float] = None
similarity_boost: Optional[float] = None
@@ -161,7 +168,7 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
voice_id: str,
model: str = "eleven_flash_v2_5",
url: str = "wss://api.elevenlabs.io",
- output_format: ElevenLabsOutputFormat = "pcm_24000",
+ sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
**kwargs,
):
@@ -185,7 +192,7 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
push_text_frames=False,
push_stop_frames=True,
stop_frame_timeout_s=2.0,
- sample_rate=sample_rate_from_output_format(output_format),
+ sample_rate=sample_rate,
**kwargs,
)
WebsocketService.__init__(self)
@@ -193,11 +200,9 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
self._api_key = api_key
self._url = url
self._settings = {
- "sample_rate": sample_rate_from_output_format(output_format),
"language": self.language_to_service_language(params.language)
if params.language
- else "en",
- "output_format": output_format,
+ else None,
"optimize_streaming_latency": params.optimize_streaming_latency,
"stability": params.stability,
"similarity_boost": params.similarity_boost,
@@ -207,6 +212,7 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
}
self.set_model_name(model)
self.set_voice(voice_id)
+ self._output_format = "" # initialized in start()
self._voice_settings = self._set_voice_settings()
# Indicates if we have sent TTSStartedFrame. It will reset to False when
@@ -250,7 +256,7 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
await self._disconnect()
await self._connect()
- async def _update_settings(self, settings: Dict[str, Any]):
+ async def _update_settings(self, settings: Mapping[str, Any]):
prev_voice = self._voice_id
await super()._update_settings(settings)
if not prev_voice == self._voice_id:
@@ -260,6 +266,7 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
async def start(self, frame: StartFrame):
await super().start(frame)
+ self._output_format = output_format_from_sample_rate(self.sample_rate)
await self._connect()
async def stop(self, frame: EndFrame):
@@ -298,20 +305,16 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
async def _connect(self):
await self._connect_websocket()
- self._receive_task = self.get_event_loop().create_task(
- self._receive_task_handler(self.push_error)
- )
- self._keepalive_task = self.get_event_loop().create_task(self._keepalive_task_handler())
+ self._receive_task = self.create_task(self._receive_task_handler(self.push_error))
+ self._keepalive_task = self.create_task(self._keepalive_task_handler())
async def _disconnect(self):
if self._receive_task:
- self._receive_task.cancel()
- await self._receive_task
+ await self.cancel_task(self._receive_task)
self._receive_task = None
if self._keepalive_task:
- self._keepalive_task.cancel()
- await self._keepalive_task
+ await self.cancel_task(self._keepalive_task)
self._keepalive_task = None
await self._disconnect_websocket()
@@ -322,7 +325,7 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
voice_id = self._voice_id
model = self.model_name
- output_format = self._settings["output_format"]
+ output_format = self._output_format
url = f"{self._url}/v1/text-to-speech/{voice_id}/stream-input?model_id={model}&output_format={output_format}&auto_mode={self._settings['auto_mode']}"
if self._settings["optimize_streaming_latency"]:
@@ -330,14 +333,16 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
# Language can only be used with the ELEVENLABS_MULTILINGUAL_MODELS
language = self._settings["language"]
- if model in ELEVENLABS_MULTILINGUAL_MODELS:
+ if model in ELEVENLABS_MULTILINGUAL_MODELS and language is not None:
url += f"&language_code={language}"
- else:
+ logger.debug(f"Using language code: {language}")
+ elif language is not None:
logger.warning(
f"Language code [{language}] not applied. Language codes can only be used with multilingual models: {', '.join(sorted(ELEVENLABS_MULTILINGUAL_MODELS))}"
)
- self._websocket = await websockets.connect(url)
+ # Set max websocket message size to 16MB for large audio responses
+ self._websocket = await websockets.connect(url, max_size=16 * 1024 * 1024)
# According to ElevenLabs, we should always start with a single space.
msg: Dict[str, Any] = {
@@ -373,7 +378,7 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
self.start_word_timestamps()
audio = base64.b64decode(msg["audio"])
- frame = TTSAudioRawFrame(audio, self._settings["sample_rate"], 1)
+ frame = TTSAudioRawFrame(audio, self.sample_rate, 1)
await self.push_frame(frame)
if msg.get("alignment"):
word_times = calculate_word_times(msg["alignment"], self._cumulative_time)
@@ -382,13 +387,8 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
async def _keepalive_task_handler(self):
while True:
- try:
- await asyncio.sleep(10)
- await self._send_text("")
- except asyncio.CancelledError:
- break
- except Exception as e:
- logger.error(f"{self} exception: {e}")
+ await asyncio.sleep(10)
+ await self._send_text("")
async def _send_text(self, text: str):
if self._websocket:
@@ -431,12 +431,12 @@ class ElevenLabsHttpTTSService(TTSService):
aiohttp_session: aiohttp ClientSession
model: Model ID (default: "eleven_flash_v2_5" for low latency)
base_url: API base URL
- output_format: Audio output format (PCM)
+ sample_rate: Output sample rate
params: Additional parameters for voice configuration
"""
class InputParams(BaseModel):
- language: Optional[Language] = Language.EN
+ language: Optional[Language] = None
optimize_streaming_latency: Optional[int] = None
stability: Optional[float] = None
similarity_boost: Optional[float] = None
@@ -451,24 +451,21 @@ class ElevenLabsHttpTTSService(TTSService):
aiohttp_session: aiohttp.ClientSession,
model: str = "eleven_flash_v2_5",
base_url: str = "https://api.elevenlabs.io",
- output_format: ElevenLabsOutputFormat = "pcm_24000",
+ sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
**kwargs,
):
- super().__init__(sample_rate=sample_rate_from_output_format(output_format), **kwargs)
+ super().__init__(sample_rate=sample_rate, **kwargs)
self._api_key = api_key
self._base_url = base_url
- self._output_format = output_format
self._params = params
self._session = aiohttp_session
self._settings = {
- "sample_rate": sample_rate_from_output_format(output_format),
"language": self.language_to_service_language(params.language)
if params.language
- else "en",
- "output_format": output_format,
+ else None,
"optimize_streaming_latency": params.optimize_streaming_latency,
"stability": params.stability,
"similarity_boost": params.similarity_boost,
@@ -477,6 +474,7 @@ class ElevenLabsHttpTTSService(TTSService):
}
self.set_model_name(model)
self.set_voice(voice_id)
+ self._output_format = "" # initialized in start()
self._voice_settings = self._set_voice_settings()
def can_generate_metrics(self) -> bool:
@@ -511,6 +509,10 @@ class ElevenLabsHttpTTSService(TTSService):
return voice_settings or None
+ async def start(self, frame: StartFrame):
+ await super().start(frame)
+ self._output_format = output_format_from_sample_rate(self.sample_rate)
+
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using ElevenLabs streaming API.
@@ -524,16 +526,22 @@ class ElevenLabsHttpTTSService(TTSService):
url = f"{self._base_url}/v1/text-to-speech/{self._voice_id}/stream"
- payload = {
+ payload: Dict[str, Union[str, Dict[str, Union[float, bool]]]] = {
"text": text,
"model_id": self._model_name,
}
if self._voice_settings:
- payload["voice_settings"] = json.dumps(self._voice_settings)
+ payload["voice_settings"] = self._voice_settings
- if self._settings["language"]:
- payload["language_code"] = self._settings["language"]
+ language = self._settings["language"]
+ if self._model_name in ELEVENLABS_MULTILINGUAL_MODELS and language:
+ payload["language_code"] = language
+ logger.debug(f"Using language code: {language}")
+ elif language:
+ logger.warning(
+ f"Language code [{language}] not applied. Language codes can only be used with multilingual models: {', '.join(sorted(ELEVENLABS_MULTILINGUAL_MODELS))}"
+ )
headers = {
"xi-api-key": self._api_key,
@@ -567,7 +575,7 @@ class ElevenLabsHttpTTSService(TTSService):
async for chunk in response.content:
if chunk:
await self.stop_ttfb_metrics()
- yield TTSAudioRawFrame(chunk, self._settings["sample_rate"], 1)
+ yield TTSAudioRawFrame(chunk, self.sample_rate, 1)
yield TTSStoppedFrame()
diff --git a/src/pipecat/services/fish.py b/src/pipecat/services/fish.py
index 710ab04ec..d61514eb2 100644
--- a/src/pipecat/services/fish.py
+++ b/src/pipecat/services/fish.py
@@ -56,7 +56,7 @@ class FishAudioTTSService(TTSService, WebsocketService):
api_key: str,
model: str, # This is the reference_id
output_format: FishAudioOutputFormat = "pcm",
- sample_rate: int = 24000,
+ sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
**kwargs,
):
@@ -70,7 +70,7 @@ class FishAudioTTSService(TTSService, WebsocketService):
self._started = False
self._settings = {
- "sample_rate": sample_rate,
+ "sample_rate": 0,
"latency": params.latency,
"format": output_format,
"prosody": {
@@ -92,6 +92,7 @@ class FishAudioTTSService(TTSService, WebsocketService):
async def start(self, frame: StartFrame):
await super().start(frame)
+ self._settings["sample_rate"] = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -104,15 +105,12 @@ class FishAudioTTSService(TTSService, WebsocketService):
async def _connect(self):
await self._connect_websocket()
- self._receive_task = self.get_event_loop().create_task(
- self._receive_task_handler(self.push_error)
- )
+ self._receive_task = self.create_task(self._receive_task_handler(self.push_error))
async def _disconnect(self):
await self._disconnect_websocket()
if self._receive_task:
- self._receive_task.cancel()
- await self._receive_task
+ await self.cancel_task(self._receive_task)
self._receive_task = None
async def _connect_websocket(self):
@@ -160,9 +158,7 @@ class FishAudioTTSService(TTSService, WebsocketService):
audio_data = msg.get("audio")
# Only process larger chunks to remove msgpack overhead
if audio_data and len(audio_data) > 1024:
- frame = TTSAudioRawFrame(
- audio_data, self._settings["sample_rate"], 1
- )
+ frame = TTSAudioRawFrame(audio_data, self.sample_rate, 1)
await self.push_frame(frame)
await self.stop_ttfb_metrics()
continue
diff --git a/src/pipecat/services/gemini_multimodal_live/events.py b/src/pipecat/services/gemini_multimodal_live/events.py
index b2107e7f8..60f688997 100644
--- a/src/pipecat/services/gemini_multimodal_live/events.py
+++ b/src/pipecat/services/gemini_multimodal_live/events.py
@@ -48,7 +48,7 @@ class AudioInputMessage(BaseModel):
realtimeInput: RealtimeInput
@classmethod
- def from_raw_audio(cls, raw_audio: bytes, sample_rate=16000) -> "AudioInputMessage":
+ def from_raw_audio(cls, raw_audio: bytes, sample_rate: int) -> "AudioInputMessage":
data = base64.b64encode(raw_audio).decode("utf-8")
return cls(
realtimeInput=RealtimeInput(
diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py
index 57753e9cd..4f26b6e9f 100644
--- a/src/pipecat/services/gemini_multimodal_live/gemini.py
+++ b/src/pipecat/services/gemini_multimodal_live/gemini.py
@@ -182,9 +182,13 @@ class GeminiMultimodalLiveLLMService(LLMService):
self._audio_input_paused = start_audio_paused
self._video_input_paused = start_video_paused
+ self._context = None
self._websocket = None
self._receive_task = None
- self._context = None
+ self._transcribe_audio_task = None
+ self._transcribe_model_audio_task = None
+ self._transcribe_audio_queue = asyncio.Queue()
+ self._transcribe_model_audio_queue = asyncio.Queue()
self._disconnecting = False
self._api_session_ready = False
@@ -199,6 +203,8 @@ class GeminiMultimodalLiveLLMService(LLMService):
self._bot_audio_buffer = bytearray()
self._bot_text_buffer = ""
+ self._sample_rate = 24000
+
self._settings = {
"frequency_penalty": params.frequency_penalty,
"max_tokens": params.max_tokens,
@@ -244,6 +250,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
async def start(self, frame: StartFrame):
await super().start(frame)
+ await self._connect()
async def stop(self, frame: EndFrame):
await super().stop(frame)
@@ -275,7 +282,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
)
await self.send_client_event(evt)
if self._transcribe_user_audio and self._context:
- asyncio.create_task(self._handle_transcribe_user_audio(audio, self._context))
+ await self._transcribe_audio_queue.put(audio)
async def _handle_transcribe_user_audio(self, audio, context):
text = await self._transcribe_audio(audio, context)
@@ -288,6 +295,10 @@ class GeminiMultimodalLiveLLMService(LLMService):
)
async def _handle_transcribe_model_audio(self, audio, context):
+ # Early return if modalities are not set to audio.
+ if self._settings["modalities"] != GeminiMultimodalModalities.AUDIO:
+ return
+
text = await self._transcribe_audio(audio, context)
logger.debug(f"[Transcription:model] {text}")
# We add user messages directly to the context. We don't do that for assistant messages,
@@ -377,17 +388,21 @@ class GeminiMultimodalLiveLLMService(LLMService):
await self._ws_send(event.model_dump(exclude_none=True))
async def _connect(self):
+ if self._websocket:
+ # Here we assume that if we have a websocket, we are connected. We
+ # handle disconnections in the send/recv code paths.
+ return
+
logger.info("Connecting to Gemini service")
try:
- if self._websocket:
- # Here we assume that if we have a websocket, we are connected. We
- # handle disconnections in the send/recv code paths.
- return
-
uri = f"wss://{self.base_url}/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent?key={self.api_key}"
logger.info(f"Connecting to {uri}")
self._websocket = await websockets.connect(uri=uri)
- self._receive_task = self.get_event_loop().create_task(self._receive_task_handler())
+ self._receive_task = self.create_task(self._receive_task_handler())
+ self._transcribe_audio_task = self.create_task(self._transcribe_audio_handler())
+ self._transcribe_model_audio_task = self.create_task(
+ self._transcribe_model_audio_handler()
+ )
config = events.Config.model_validate(
{
"setup": {
@@ -437,12 +452,14 @@ class GeminiMultimodalLiveLLMService(LLMService):
await self._websocket.close()
self._websocket = None
if self._receive_task:
- self._receive_task.cancel()
- try:
- await asyncio.wait_for(self._receive_task, timeout=1.0)
- except asyncio.TimeoutError:
- logger.warning("Timed out waiting for receive task to finish")
+ await self.cancel_task(self._receive_task, timeout=1.0)
self._receive_task = None
+ if self._transcribe_audio_task:
+ await self.cancel_task(self._transcribe_audio_task)
+ self._transcribe_audio_task = None
+ if self._transcribe_model_audio_task:
+ await self.cancel_task(self._transcribe_model_audio_task)
+ self._transcribe_model_audio_task = None
self._disconnecting = False
except Exception as e:
logger.error(f"{self} error disconnecting: {e}")
@@ -450,9 +467,8 @@ class GeminiMultimodalLiveLLMService(LLMService):
async def _ws_send(self, message):
# logger.debug(f"Sending message to websocket: {message}")
try:
- if not self._websocket:
- await self._connect()
- await self._websocket.send(json.dumps(message))
+ if self._websocket:
+ await self._websocket.send(json.dumps(message))
except Exception as e:
if self._disconnecting:
return
@@ -469,32 +485,35 @@ class GeminiMultimodalLiveLLMService(LLMService):
#
async def _receive_task_handler(self):
- try:
- async for message in self._websocket:
- evt = events.parse_server_event(message)
- # logger.debug(f"Received event: {message[:500]}")
- # logger.debug(f"Received event: {evt}")
+ async for message in self._websocket:
+ evt = events.parse_server_event(message)
+ # logger.debug(f"Received event: {message[:500]}")
+ # logger.debug(f"Received event: {evt}")
- if evt.setupComplete:
- await self._handle_evt_setup_complete(evt)
- elif evt.serverContent and evt.serverContent.modelTurn:
- await self._handle_evt_model_turn(evt)
- elif evt.serverContent and evt.serverContent.turnComplete:
- await self._handle_evt_turn_complete(evt)
- elif evt.toolCall:
- await self._handle_evt_tool_call(evt)
+ if evt.setupComplete:
+ await self._handle_evt_setup_complete(evt)
+ elif evt.serverContent and evt.serverContent.modelTurn:
+ await self._handle_evt_model_turn(evt)
+ elif evt.serverContent and evt.serverContent.turnComplete:
+ await self._handle_evt_turn_complete(evt)
+ elif evt.toolCall:
+ await self._handle_evt_tool_call(evt)
+ elif False: # !!! todo: error events?
+ await self._handle_evt_error(evt)
+ # errors are fatal, so exit the receive loop
+ return
+ else:
+ pass
- elif False: # !!! todo: error events?
- await self._handle_evt_error(evt)
- # errors are fatal, so exit the receive loop
- return
+ async def _transcribe_audio_handler(self):
+ while True:
+ audio = await self._transcribe_audio_queue.get()
+ await self._handle_transcribe_user_audio(audio, self._context)
- else:
- pass
- except asyncio.CancelledError:
- logger.debug("websocket receive task cancelled")
- except Exception as e:
- logger.error(f"{self} exception: {e}")
+ async def _transcribe_model_audio_handler(self):
+ while True:
+ audio = await self._transcribe_model_audio_queue.get()
+ await self._handle_transcribe_model_audio(audio, self._context)
#
#
@@ -504,7 +523,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
if self._audio_input_paused:
return
# Send all audio to Gemini
- evt = events.AudioInputMessage.from_raw_audio(frame.audio)
+ evt = events.AudioInputMessage.from_raw_audio(frame.audio, frame.sample_rate)
await self.send_client_event(evt)
# Manage a buffer of audio to use for transcription
audio = frame.audio
@@ -633,7 +652,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
inline_data = part.inlineData
if not inline_data:
return
- if inline_data.mimeType != "audio/pcm;rate=24000":
+ if inline_data.mimeType != f"audio/pcm;rate={self._sample_rate}":
logger.warning(f"Unrecognized server_content format {inline_data.mimeType}")
return
@@ -648,7 +667,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
self._bot_audio_buffer.extend(audio)
frame = TTSAudioRawFrame(
audio=audio,
- sample_rate=24000,
+ sample_rate=self._sample_rate,
num_channels=1,
)
await self.push_frame(frame)
@@ -675,7 +694,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
self._bot_text_buffer = ""
if audio and self._transcribe_model_audio and self._context:
- asyncio.create_task(self._handle_transcribe_model_audio(audio, self._context))
+ await self._transcribe_model_audio_queue.put(audio)
elif text:
await self.push_frame(LLMFullResponseEndFrame())
diff --git a/src/pipecat/services/gladia.py b/src/pipecat/services/gladia.py
index 3487be5ae..3a91c62fa 100644
--- a/src/pipecat/services/gladia.py
+++ b/src/pipecat/services/gladia.py
@@ -131,7 +131,6 @@ def language_to_gladia_language(language: Language) -> str | None:
class GladiaSTTService(STTService):
class InputParams(BaseModel):
- sample_rate: Optional[int] = 16000
language: Optional[Language] = Language.EN
endpointing: Optional[float] = 0.2
maximum_duration_without_endpointing: Optional[int] = 10
@@ -144,17 +143,18 @@ class GladiaSTTService(STTService):
api_key: str,
url: str = "https://api.gladia.io/v2/live",
confidence: float = 0.5,
+ sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
**kwargs,
):
- super().__init__(**kwargs)
+ super().__init__(sample_rate=sample_rate, **kwargs)
self._api_key = api_key
self._url = url
self._settings = {
"encoding": "wav/pcm",
"bit_depth": 16,
- "sample_rate": params.sample_rate,
+ "sample_rate": 0,
"channels": 1,
"language_config": {
"languages": [self.language_to_service_language(params.language)]
@@ -178,18 +178,21 @@ class GladiaSTTService(STTService):
async def start(self, frame: StartFrame):
await super().start(frame)
+ self._settings["sample_rate"] = self.sample_rate
response = await self._setup_gladia()
self._websocket = await websockets.connect(response["url"])
- self._receive_task = self.get_event_loop().create_task(self._receive_task_handler())
+ self._receive_task = self.create_task(self._receive_task_handler())
async def stop(self, frame: EndFrame):
await super().stop(frame)
await self._send_stop_recording()
await self._websocket.close()
+ await self.wait_for_task(self._receive_task)
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
await self._websocket.close()
+ await self.cancel_task(self._receive_task)
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
await self.start_processing_metrics()
diff --git a/src/pipecat/services/google/__init__.py b/src/pipecat/services/google/__init__.py
new file mode 100644
index 000000000..1b81980e6
--- /dev/null
+++ b/src/pipecat/services/google/__init__.py
@@ -0,0 +1,2 @@
+from .frames import LLMSearchResponseFrame
+from .google import *
diff --git a/src/pipecat/services/google/frames.py b/src/pipecat/services/google/frames.py
new file mode 100644
index 000000000..700a39a7e
--- /dev/null
+++ b/src/pipecat/services/google/frames.py
@@ -0,0 +1,33 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from pipecat.frames.frames import DataFrame
+
+
+@dataclass
+class LLMSearchResult:
+ text: str
+ confidence: List[float] = field(default_factory=list)
+
+
+@dataclass
+class LLMSearchOrigin:
+ site_uri: Optional[str] = None
+ site_title: Optional[str] = None
+ results: List[LLMSearchResult] = field(default_factory=list)
+
+
+@dataclass
+class LLMSearchResponseFrame(DataFrame):
+ search_result: Optional[str] = None
+ rendered_content: Optional[str] = None
+ origins: List[LLMSearchOrigin] = field(default_factory=list)
+
+ def __str__(self):
+ return f"LLMSearchResponseFrame(search_result={self.search_result}, origins={self.origins})"
diff --git a/src/pipecat/services/google.py b/src/pipecat/services/google/google.py
similarity index 78%
rename from src/pipecat/services/google.py
rename to src/pipecat/services/google/google.py
index 4e4db5990..971bf61c0 100644
--- a/src/pipecat/services/google.py
+++ b/src/pipecat/services/google/google.py
@@ -29,6 +29,7 @@ from pipecat.frames.frames import (
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
+ URLImageRawFrame,
VisionImageRawFrame,
)
from pipecat.metrics.metrics import LLMTokenUsage
@@ -37,7 +38,8 @@ from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContextFrame,
)
from pipecat.processors.frame_processor import FrameDirection
-from pipecat.services.ai_services import LLMService, TTSService
+from pipecat.services.ai_services import ImageGenService, LLMService, TTSService
+from pipecat.services.google.frames import LLMSearchResponseFrame
from pipecat.services.openai import (
OpenAIAssistantContextAggregator,
OpenAIUserContextAggregator,
@@ -48,7 +50,9 @@ from pipecat.utils.time import time_now_iso8601
try:
import google.ai.generativelanguage as glm
import google.generativeai as gai
+ from google import genai
from google.cloud import texttospeech_v1
+ from google.genai import types
from google.generativeai.types import GenerationConfig
from google.oauth2 import service_account
except ModuleNotFoundError as e:
@@ -563,23 +567,56 @@ class GoogleLLMContext(OpenAILLMContext):
return [msg]
def _restructure_from_openai_messages(self):
+ """Restructures messages to ensure proper Google format and message ordering.
+
+ This method handles conversion of OpenAI-formatted messages to Google format,
+ with special handling for function calls, function responses, and system messages.
+ System messages are added back to the context as user messages when needed.
+
+ The final message order is preserved as:
+ 1. Function calls (from model)
+ 2. Function responses (from user)
+ 3. Text messages (converted from system messages)
+
+ Note:
+ System messages are only added back when there are no regular text
+ messages in the context, ensuring proper conversation continuity
+ after function calls.
+ """
self.system_message = None
- # first, map across self._messages calling self.from_standard_message(m) to modify messages in place
- try:
- self._messages[:] = [
- msg
- for msg in (self.from_standard_message(m) for m in self._messages)
- if msg is not None
- ]
- # We might have been given a messages list with only a system message. If so, let's put that back in
- # the messages list as a user message.
- if self.system_message and not self._messages:
- self.add_message(
- glm.Content(role="user", parts=[glm.Part(text=self.system_message)])
- )
- except Exception as e:
- logger.error(f"Error mapping messages: {e}")
- # iterate over messages and remove any messages that have an empty content list
+ converted_messages = []
+
+ # Process each message, preserving Google-formatted messages and converting others
+ for message in self._messages:
+ if isinstance(message, glm.Content):
+ # Keep existing Google-formatted messages (e.g., function calls/responses)
+ converted_messages.append(message)
+ continue
+
+ # Convert OpenAI format to Google format, system messages return None
+ converted = self.from_standard_message(message)
+ if converted is not None:
+ converted_messages.append(converted)
+
+ # Update message list
+ self._messages[:] = converted_messages
+
+ # Check if we only have function-related messages (no regular text)
+ has_regular_messages = any(
+ len(msg.parts) == 1
+ and hasattr(msg.parts[0], "text")
+ and not hasattr(msg.parts[0], "function_call")
+ and not hasattr(msg.parts[0], "function_response")
+ for msg in self._messages
+ )
+
+ # Add system message back as a user message if we only have function messages
+ if self.system_message and not has_regular_messages:
+ self._messages.append(
+ glm.Content(role="user", parts=[glm.Part(text=self.system_message)])
+ )
+
+ # Remove any empty messages
self._messages = [m for m in self._messages if m.parts]
@@ -639,6 +676,9 @@ class GoogleLLMService(LLMService):
completion_tokens = 0
total_tokens = 0
+ grounding_metadata = None
+ search_result = ""
+
try:
logger.debug(
# f"Generating chat: {self._system_instruction} | {context.get_messages_for_logging()}"
@@ -674,7 +714,6 @@ class GoogleLLMService(LLMService):
tool_config = None
if self._tool_config:
tool_config = self._tool_config
-
response = await self._client.generate_content_async(
contents=messages,
tools=tools,
@@ -698,9 +737,10 @@ class GoogleLLMService(LLMService):
try:
for c in chunk.parts:
if c.text:
+ search_result += c.text
await self.push_frame(LLMTextFrame(c.text))
elif c.function_call:
- logger.debug(f"!!! Function call: {c.function_call}")
+ logger.debug(f"Function call: {c.function_call}")
args = type(c.function_call).to_dict(c.function_call).get("args", {})
await self.call_function(
context=context,
@@ -708,6 +748,63 @@ class GoogleLLMService(LLMService):
function_name=c.function_call.name,
arguments=args,
)
+ # Handle grounding metadata
+ # It seems only the last chunk that we receive may contain this information
+ # If the response doesn't include groundingMetadata, this means the response wasn't grounded.
+ if chunk.candidates:
+ for candidate in chunk.candidates:
+ # logger.debug(f"candidate received: {candidate}")
+ # Extract grounding metadata
+ grounding_metadata = (
+ {
+ "rendered_content": getattr(
+ getattr(candidate, "grounding_metadata", None),
+ "search_entry_point",
+ None,
+ ).rendered_content
+ if hasattr(
+ getattr(candidate, "grounding_metadata", None),
+ "search_entry_point",
+ )
+ else None,
+ "origins": [
+ {
+ "site_uri": getattr(grounding_chunk.web, "uri", None),
+ "site_title": getattr(
+ grounding_chunk.web, "title", None
+ ),
+ "results": [
+ {
+ "text": getattr(
+ grounding_support.segment, "text", ""
+ ),
+ "confidence": getattr(
+ grounding_support, "confidence_scores", None
+ ),
+ }
+ for grounding_support in getattr(
+ getattr(candidate, "grounding_metadata", None),
+ "grounding_supports",
+ [],
+ )
+ if index
+ in getattr(
+ grounding_support, "grounding_chunk_indices", []
+ )
+ ],
+ }
+ for index, grounding_chunk in enumerate(
+ getattr(
+ getattr(candidate, "grounding_metadata", None),
+ "grounding_chunks",
+ [],
+ )
+ )
+ ],
+ }
+ if getattr(candidate, "grounding_metadata", None)
+ else None
+ )
except Exception as e:
# Google LLMs seem to flag safety issues a lot!
if chunk.candidates[0].finish_reason == 3:
@@ -720,6 +817,14 @@ class GoogleLLMService(LLMService):
except Exception as e:
logger.exception(f"{self} exception: {e}")
finally:
+ if grounding_metadata is not None and isinstance(grounding_metadata, dict):
+ llm_search_frame = LLMSearchResponseFrame(
+ search_result=search_result,
+ origins=grounding_metadata["origins"],
+ rendered_content=grounding_metadata["rendered_content"],
+ )
+ await self.push_frame(llm_search_frame)
+
await self.start_llm_usage_metrics(
LLMTokenUsage(
prompt_tokens=prompt_tokens,
@@ -778,14 +883,13 @@ class GoogleTTSService(TTSService):
credentials: Optional[str] = None,
credentials_path: Optional[str] = None,
voice_id: str = "en-US-Neural2-A",
- sample_rate: int = 24000,
+ sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
self._settings = {
- "sample_rate": sample_rate,
"pitch": params.pitch,
"rate": params.rate,
"volume": params.volume,
@@ -891,7 +995,7 @@ class GoogleTTSService(TTSService):
)
audio_config = texttospeech_v1.AudioConfig(
audio_encoding=texttospeech_v1.AudioEncoding.LINEAR16,
- sample_rate_hertz=self._settings["sample_rate"],
+ sample_rate_hertz=self.sample_rate,
)
request = texttospeech_v1.SynthesizeSpeechRequest(
@@ -914,7 +1018,7 @@ class GoogleTTSService(TTSService):
if not chunk:
break
await self.stop_ttfb_metrics()
- frame = TTSAudioRawFrame(chunk, self._settings["sample_rate"], 1)
+ frame = TTSAudioRawFrame(chunk, self.sample_rate, 1)
yield frame
await asyncio.sleep(0) # Allow other tasks to run
@@ -926,3 +1030,70 @@ class GoogleTTSService(TTSService):
yield ErrorFrame(error=error_message)
finally:
yield TTSStoppedFrame()
+
+
+class GoogleImageGenService(ImageGenService):
+ class InputParams(BaseModel):
+ number_of_images: int = Field(default=1, ge=1, le=8)
+ model: str = Field(default="imagen-3.0-generate-002")
+ negative_prompt: str = Field(default=None)
+
+ def __init__(
+ self,
+ *,
+ params: InputParams = InputParams(),
+ api_key: str,
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+ self.set_model_name(params.model)
+ self._params = params
+ self._client = genai.Client(api_key=api_key)
+
+ def can_generate_metrics(self) -> bool:
+ return True
+
+ async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]:
+ """Generate images from a text prompt using Google's Imagen model.
+
+ Args:
+ prompt (str): The text description to generate images from.
+
+ Yields:
+ Frame: Generated image frames or error frames.
+ """
+ logger.debug(f"Generating image from prompt: {prompt}")
+ await self.start_ttfb_metrics()
+
+ try:
+ response = await self._client.aio.models.generate_images(
+ model=self._params.model,
+ prompt=prompt,
+ config=types.GenerateImagesConfig(
+ number_of_images=self._params.number_of_images,
+ negative_prompt=self._params.negative_prompt,
+ ),
+ )
+ await self.stop_ttfb_metrics()
+
+ if not response or not response.generated_images:
+ logger.error(f"{self} error: image generation failed")
+ yield ErrorFrame("Image generation failed")
+ return
+
+ for img_response in response.generated_images:
+ # Google returns the image data directly
+ image_bytes = img_response.image.image_bytes
+ image = Image.open(io.BytesIO(image_bytes))
+
+ frame = URLImageRawFrame(
+ url=None, # Google doesn't provide URLs, only image data
+ image=image.tobytes(),
+ size=image.size,
+ format=image.format,
+ )
+ yield frame
+
+ except Exception as e:
+ logger.error(f"{self} error generating image: {e}")
+ yield ErrorFrame(f"Image generation error: {str(e)}")
diff --git a/src/pipecat/services/lmnt.py b/src/pipecat/services/lmnt.py
index 6f654293a..e40dc2ce9 100644
--- a/src/pipecat/services/lmnt.py
+++ b/src/pipecat/services/lmnt.py
@@ -5,7 +5,7 @@
#
import json
-from typing import AsyncGenerator
+from typing import AsyncGenerator, Optional
from loguru import logger
@@ -66,7 +66,7 @@ class LmntTTSService(TTSService, WebsocketService):
*,
api_key: str,
voice_id: str,
- sample_rate: int = 24000,
+ sample_rate: Optional[int] = None,
language: Language = Language.EN,
**kwargs,
):
@@ -81,7 +81,6 @@ class LmntTTSService(TTSService, WebsocketService):
self._api_key = api_key
self._voice_id = voice_id
self._settings = {
- "sample_rate": sample_rate,
"language": self.language_to_service_language(language),
"format": "raw", # Use raw format for direct PCM data
}
@@ -113,16 +112,13 @@ class LmntTTSService(TTSService, WebsocketService):
async def _connect(self):
await self._connect_websocket()
- self._receive_task = self.get_event_loop().create_task(
- self._receive_task_handler(self.push_error)
- )
+ self._receive_task = self.create_task(self._receive_task_handler(self.push_error))
async def _disconnect(self):
await self._disconnect_websocket()
if self._receive_task:
- self._receive_task.cancel()
- await self._receive_task
+ await self.cancel_task(self._receive_task)
self._receive_task = None
async def _connect_websocket(self):
@@ -135,7 +131,7 @@ class LmntTTSService(TTSService, WebsocketService):
"X-API-Key": self._api_key,
"voice": self._voice_id,
"format": self._settings["format"],
- "sample_rate": self._settings["sample_rate"],
+ "sample_rate": self.sample_rate,
"language": self._settings["language"],
}
@@ -178,7 +174,7 @@ class LmntTTSService(TTSService, WebsocketService):
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(
audio=message,
- sample_rate=self._settings["sample_rate"],
+ sample_rate=self.sample_rate,
num_channels=1,
)
await self.push_frame(frame)
diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py
index 19c8f6d99..519ec800d 100644
--- a/src/pipecat/services/openai.py
+++ b/src/pipecat/services/openai.py
@@ -28,6 +28,7 @@ from pipecat.frames.frames import (
LLMTextFrame,
LLMUpdateSettingsFrame,
OpenAILLMContextAssistantTimestampFrame,
+ StartFrame,
StartInterruptionFrame,
TTSAudioRawFrame,
TTSStartedFrame,
@@ -116,6 +117,8 @@ class BaseOpenAILLMService(LLMService):
model: str,
api_key=None,
base_url=None,
+ organization=None,
+ project=None,
params: InputParams = InputParams(),
**kwargs,
):
@@ -131,12 +134,16 @@ class BaseOpenAILLMService(LLMService):
"extra": params.extra if isinstance(params.extra, dict) else {},
}
self.set_model_name(model)
- self._client = self.create_client(api_key=api_key, base_url=base_url, **kwargs)
+ self._client = self.create_client(
+ api_key=api_key, base_url=base_url, organization=organization, project=project, **kwargs
+ )
- def create_client(self, api_key=None, base_url=None, **kwargs):
+ def create_client(self, api_key=None, base_url=None, organization=None, project=None, **kwargs):
return AsyncOpenAI(
api_key=api_key,
base_url=base_url,
+ organization=organization,
+ project=project,
http_client=DefaultAsyncHttpxClient(
limits=httpx.Limits(
max_keepalive_connections=100, max_connections=1000, keepalive_expiry=None
@@ -221,7 +228,7 @@ class BaseOpenAILLMService(LLMService):
)
await self.start_llm_usage_metrics(tokens)
- if len(chunk.choices) == 0:
+ if chunk.choices is None or len(chunk.choices) == 0:
continue
await self.stop_ttfb_metrics()
@@ -406,20 +413,24 @@ class OpenAITTSService(TTSService):
The service returns PCM-encoded audio at the specified sample rate.
"""
+ OPENAI_SAMPLE_RATE = 24000 # OpenAI TTS always outputs at 24kHz
+
def __init__(
self,
*,
- api_key: str | None = None,
+ api_key: Optional[str] = None,
voice: str = "alloy",
model: Literal["tts-1", "tts-1-hd"] = "tts-1",
- sample_rate: int = 24000,
+ sample_rate: Optional[int] = None,
**kwargs,
):
+ if sample_rate and sample_rate != self.OPENAI_SAMPLE_RATE:
+ logger.warning(
+ f"OpenAI TTS only supports {self.OPENAI_SAMPLE_RATE}Hz sample rate. "
+ f"Current rate of {self.sample_rate}Hz may cause issues."
+ )
super().__init__(sample_rate=sample_rate, **kwargs)
- self._settings = {
- "sample_rate": sample_rate,
- }
self.set_model_name(model)
self.set_voice(voice)
@@ -432,6 +443,14 @@ class OpenAITTSService(TTSService):
logger.info(f"Switching TTS model to: [{model}]")
self.set_model_name(model)
+ async def start(self, frame: StartFrame):
+ await super().start(frame)
+ if self.sample_rate != self.OPENAI_SAMPLE_RATE:
+ logger.warning(
+ f"OpenAI TTS requires {self.OPENAI_SAMPLE_RATE}Hz sample rate. "
+ f"Current rate of {self.sample_rate}Hz may cause issues."
+ )
+
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
try:
@@ -459,7 +478,7 @@ class OpenAITTSService(TTSService):
async for chunk in r.iter_bytes(8192):
if len(chunk) > 0:
await self.stop_ttfb_metrics()
- frame = TTSAudioRawFrame(chunk, self._settings["sample_rate"], 1)
+ frame = TTSAudioRawFrame(chunk, self.sample_rate, 1)
yield frame
yield TTSStoppedFrame()
except BadRequestError as e:
diff --git a/src/pipecat/services/openai_realtime_beta/context.py b/src/pipecat/services/openai_realtime_beta/context.py
index 91f01d5a1..da287194a 100644
--- a/src/pipecat/services/openai_realtime_beta/context.py
+++ b/src/pipecat/services/openai_realtime_beta/context.py
@@ -6,10 +6,16 @@
import copy
import json
+from typing import Optional
from loguru import logger
-from pipecat.frames.frames import Frame, LLMMessagesUpdateFrame, LLMSetToolsFrame
+from pipecat.frames.frames import (
+ Frame,
+ FunctionCallResultProperties,
+ LLMMessagesUpdateFrame,
+ LLMSetToolsFrame,
+)
from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContext,
OpenAILLMContextFrame,
@@ -174,10 +180,13 @@ class OpenAIRealtimeAssistantContextAggregator(OpenAIAssistantContextAggregator)
if not self._function_call_result:
return
+ properties: Optional[FunctionCallResultProperties] = None
+
self._reset()
try:
run_llm = True
frame = self._function_call_result
+ properties = frame.properties
self._function_call_result = None
if frame.result:
# The "tool_call" message from the LLM that triggered the function call
@@ -211,11 +220,20 @@ class OpenAIRealtimeAssistantContextAggregator(OpenAIAssistantContextAggregator)
await self._user_context_aggregator.push_frame(
RealtimeFunctionCallResultFrame(result_frame=frame)
)
- run_llm = frame.run_llm
+ if properties and properties.run_llm is not None:
+ # If the tool call result has a run_llm property, use it
+ run_llm = properties.run_llm
+ else:
+ # Default behavior is to run the LLM if there are no function calls in progress
+ run_llm = not bool(self._function_calls_in_progress)
if run_llm:
await self._user_context_aggregator.push_context_frame()
+ # Emit the on_context_updated callback once the function call result is added to the context
+ if properties and properties.on_context_updated is not None:
+ await properties.on_context_updated()
+
frame = OpenAILLMContextFrame(self._context)
await self.push_frame(frame)
diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py
index c1d7c4a4d..a42ed7b9a 100644
--- a/src/pipecat/services/openai_realtime_beta/openai.py
+++ b/src/pipecat/services/openai_realtime_beta/openai.py
@@ -167,8 +167,11 @@ class OpenAIRealtimeBetaLLMService(LLMService):
either the wall clock time or the actual audio duration to prevent invalid truncation
requests.
"""
+ if not self._current_audio_response:
+ return
+
# if the bot is still speaking, truncate the last message
- if self._current_audio_response:
+ try:
current = self._current_audio_response
self._current_audio_response = None
@@ -179,6 +182,11 @@ class OpenAIRealtimeBetaLLMService(LLMService):
elapsed_ms = int(time.time() * 1000 - current.start_time_ms)
truncate_ms = min(elapsed_ms, audio_duration_ms)
+ logger.trace(
+ f"Truncating audio: duration={audio_duration_ms}ms, "
+ f"elapsed={elapsed_ms}ms, truncate={truncate_ms}ms"
+ )
+
await self.send_client_event(
events.ConversationItemTruncateEvent(
item_id=current.item_id,
@@ -186,6 +194,9 @@ class OpenAIRealtimeBetaLLMService(LLMService):
audio_end_ms=truncate_ms,
)
)
+ except Exception as e:
+ # Log warning and don't re-raise - allow session to continue
+ logger.warning(f"Audio truncation failed (non-fatal): {e}")
#
# frame processing
@@ -266,7 +277,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
"OpenAI-Beta": "realtime=v1",
},
)
- self._receive_task = self.get_event_loop().create_task(self._receive_task_handler())
+ self._receive_task = self.create_task(self._receive_task_handler())
except Exception as e:
logger.error(f"{self} initialization error: {e}")
self._websocket = None
@@ -280,11 +291,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
await self._websocket.close()
self._websocket = None
if self._receive_task:
- self._receive_task.cancel()
- try:
- await asyncio.wait_for(self._receive_task, timeout=1.0)
- except asyncio.TimeoutError:
- logger.warning("Timed out waiting for receive task to finish")
+ await self.cancel_task(self._receive_task, timeout=1.0)
self._receive_task = None
self._disconnecting = False
except Exception as e:
@@ -321,40 +328,32 @@ class OpenAIRealtimeBetaLLMService(LLMService):
#
async def _receive_task_handler(self):
- try:
- async for message in self._websocket:
- evt = events.parse_server_event(message)
- if evt.type == "session.created":
- await self._handle_evt_session_created(evt)
- elif evt.type == "session.updated":
- await self._handle_evt_session_updated(evt)
- elif evt.type == "response.audio.delta":
- await self._handle_evt_audio_delta(evt)
- elif evt.type == "response.audio.done":
- await self._handle_evt_audio_done(evt)
- elif evt.type == "conversation.item.created":
- await self._handle_evt_conversation_item_created(evt)
- elif evt.type == "conversation.item.input_audio_transcription.completed":
- await self.handle_evt_input_audio_transcription_completed(evt)
- elif evt.type == "response.done":
- await self._handle_evt_response_done(evt)
- elif evt.type == "input_audio_buffer.speech_started":
- await self._handle_evt_speech_started(evt)
- elif evt.type == "input_audio_buffer.speech_stopped":
- await self._handle_evt_speech_stopped(evt)
- elif evt.type == "response.audio_transcript.delta":
- await self._handle_evt_audio_transcript_delta(evt)
- elif evt.type == "error":
- await self._handle_evt_error(evt)
- # errors are fatal, so exit the receive loop
- return
-
- else:
- pass
- except asyncio.CancelledError:
- logger.debug("websocket receive task cancelled")
- except Exception as e:
- logger.error(f"{self} exception: {e}")
+ async for message in self._websocket:
+ evt = events.parse_server_event(message)
+ if evt.type == "session.created":
+ await self._handle_evt_session_created(evt)
+ elif evt.type == "session.updated":
+ await self._handle_evt_session_updated(evt)
+ elif evt.type == "response.audio.delta":
+ await self._handle_evt_audio_delta(evt)
+ elif evt.type == "response.audio.done":
+ await self._handle_evt_audio_done(evt)
+ elif evt.type == "conversation.item.created":
+ await self._handle_evt_conversation_item_created(evt)
+ elif evt.type == "conversation.item.input_audio_transcription.completed":
+ await self.handle_evt_input_audio_transcription_completed(evt)
+ elif evt.type == "response.done":
+ await self._handle_evt_response_done(evt)
+ elif evt.type == "input_audio_buffer.speech_started":
+ await self._handle_evt_speech_started(evt)
+ elif evt.type == "input_audio_buffer.speech_stopped":
+ await self._handle_evt_speech_stopped(evt)
+ elif evt.type == "response.audio_transcript.delta":
+ await self._handle_evt_audio_transcript_delta(evt)
+ elif evt.type == "error":
+ await self._handle_evt_error(evt)
+ # errors are fatal, so exit the receive loop
+ return
async def _handle_evt_session_created(self, evt):
# session.created is received right after connecting. Send a message
diff --git a/src/pipecat/services/playht.py b/src/pipecat/services/playht.py
index 46b724a4d..74caff303 100644
--- a/src/pipecat/services/playht.py
+++ b/src/pipecat/services/playht.py
@@ -113,7 +113,7 @@ class PlayHTTTSService(TTSService, WebsocketService):
user_id: str,
voice_url: str,
voice_engine: str = "Play3.0-mini",
- sample_rate: int = 24000,
+ sample_rate: Optional[int] = None,
output_format: str = "wav",
params: InputParams = InputParams(),
**kwargs,
@@ -132,7 +132,6 @@ class PlayHTTTSService(TTSService, WebsocketService):
self._request_id = None
self._settings = {
- "sample_rate": sample_rate,
"language": self.language_to_service_language(params.language)
if params.language
else "english",
@@ -165,16 +164,13 @@ class PlayHTTTSService(TTSService, WebsocketService):
async def _connect(self):
await self._connect_websocket()
- self._receive_task = self.get_event_loop().create_task(
- self._receive_task_handler(self.push_error)
- )
+ self._receive_task = self.create_task(self._receive_task_handler(self.push_error))
async def _disconnect(self):
await self._disconnect_websocket()
if self._receive_task:
- self._receive_task.cancel()
- await self._receive_task
+ await self.cancel_task(self._receive_task)
self._receive_task = None
async def _connect_websocket(self):
@@ -253,7 +249,7 @@ class PlayHTTTSService(TTSService, WebsocketService):
if message.startswith(b"RIFF"):
continue
await self.stop_ttfb_metrics()
- frame = TTSAudioRawFrame(message, self._settings["sample_rate"], 1)
+ frame = TTSAudioRawFrame(message, self.sample_rate, 1)
await self.push_frame(frame)
else:
logger.debug(f"Received text message: {message}")
@@ -304,7 +300,7 @@ class PlayHTTTSService(TTSService, WebsocketService):
"voice": self._voice_id,
"voice_engine": self._settings["voice_engine"],
"output_format": self._settings["output_format"],
- "sample_rate": self._settings["sample_rate"],
+ "sample_rate": self.sample_rate,
"language": self._settings["language"],
"speed": self._settings["speed"],
"seed": self._settings["seed"],
@@ -342,7 +338,7 @@ class PlayHTHttpTTSService(TTSService):
user_id: str,
voice_url: str,
voice_engine: str = "Play3.0-mini-http", # Options: Play3.0-mini-http, Play3.0-mini-ws
- sample_rate: int = 24000,
+ sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
**kwargs,
):
@@ -356,7 +352,6 @@ class PlayHTHttpTTSService(TTSService):
api_key=self._api_key,
)
self._settings = {
- "sample_rate": sample_rate,
"language": self.language_to_service_language(params.language)
if params.language
else "english",
@@ -368,6 +363,11 @@ class PlayHTHttpTTSService(TTSService):
self.set_model_name(voice_engine)
self.set_voice(voice_url)
+ async def start(self, frame: StartFrame):
+ await super().start(frame)
+ self._settings["sample_rate"] = self.sample_rate
+
+ def _create_options(self) -> TTSOptions:
language_str = self._settings["language"]
playht_language = None
if language_str:
@@ -377,10 +377,10 @@ class PlayHTHttpTTSService(TTSService):
playht_language = lang
break
- self._options = TTSOptions(
+ return TTSOptions(
voice=self._voice_id,
language=playht_language,
- sample_rate=self._settings["sample_rate"],
+ sample_rate=self.sample_rate,
format=self._settings["format"],
speed=self._settings["speed"],
seed=self._settings["seed"],
@@ -396,13 +396,14 @@ class PlayHTHttpTTSService(TTSService):
logger.debug(f"Generating TTS: [{text}]")
try:
+ options = self._create_options()
b = bytearray()
in_header = True
await self.start_ttfb_metrics()
playht_gen = self._client.tts(
- text, voice_engine=self._settings["voice_engine"], options=self._options
+ text, voice_engine=self._settings["voice_engine"], options=options
)
await self.start_tts_usage_metrics(text)
@@ -425,7 +426,7 @@ class PlayHTHttpTTSService(TTSService):
else:
if len(chunk):
await self.stop_ttfb_metrics()
- frame = TTSAudioRawFrame(chunk, self._settings["sample_rate"], 1)
+ frame = TTSAudioRawFrame(chunk, self.sample_rate, 1)
yield frame
yield TTSStoppedFrame()
except Exception as e:
diff --git a/src/pipecat/services/rime.py b/src/pipecat/services/rime.py
index 444461f6b..51bc7253d 100644
--- a/src/pipecat/services/rime.py
+++ b/src/pipecat/services/rime.py
@@ -34,7 +34,7 @@ class RimeHttpTTSService(TTSService):
api_key: str,
voice_id: str = "eva",
model: str = "mist",
- sample_rate: int = 24000,
+ sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
**kwargs,
):
@@ -43,7 +43,6 @@ class RimeHttpTTSService(TTSService):
self._api_key = api_key
self._base_url = "https://users.rime.ai/v1/rime-tts"
self._settings = {
- "samplingRate": sample_rate,
"speedAlpha": params.speed_alpha,
"reduceLatency": params.reduce_latency,
"pauseBetweenBrackets": params.pause_between_brackets,
@@ -71,6 +70,7 @@ class RimeHttpTTSService(TTSService):
payload["text"] = text
payload["speaker"] = self._voice_id
payload["modelId"] = self._model_name
+ payload["samplingRate"] = self.sample_rate
try:
await self.start_ttfb_metrics()
@@ -96,7 +96,7 @@ class RimeHttpTTSService(TTSService):
first_chunk = False
if chunk:
- frame = TTSAudioRawFrame(chunk, self._settings["samplingRate"], 1)
+ frame = TTSAudioRawFrame(chunk, self.sample_rate, 1)
yield frame
yield TTSStoppedFrame()
diff --git a/src/pipecat/services/riva.py b/src/pipecat/services/riva.py
index 662ac21f5..458cb4919 100644
--- a/src/pipecat/services/riva.py
+++ b/src/pipecat/services/riva.py
@@ -49,7 +49,7 @@ class FastPitchTTSService(TTSService):
api_key: str,
server: str = "grpc.nvcf.nvidia.com:443",
voice_id: str = "English-US.Female-1",
- sample_rate: int = 24000,
+ sample_rate: Optional[int] = None,
function_id: str = "0149dedb-2be8-4195-b9a0-e57e0e14f972",
params: InputParams = InputParams(),
**kwargs,
@@ -57,7 +57,6 @@ class FastPitchTTSService(TTSService):
super().__init__(sample_rate=sample_rate, **kwargs)
self._api_key = api_key
self._voice_id = voice_id
- self._sample_rate = sample_rate
self._language_code = params.language
self._quality = params.quality
@@ -87,7 +86,7 @@ class FastPitchTTSService(TTSService):
text,
self._voice_id,
self._language_code,
- sample_rate_hz=self._sample_rate,
+ sample_rate_hz=self.sample_rate,
audio_prompt_file=None,
quality=self._quality,
custom_dictionary={},
@@ -114,7 +113,7 @@ class FastPitchTTSService(TTSService):
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(
audio=resp.audio,
- sample_rate=self._sample_rate,
+ sample_rate=self.sample_rate,
num_channels=1,
)
yield frame
@@ -136,10 +135,11 @@ class ParakeetSTTService(STTService):
api_key: str,
server: str = "grpc.nvcf.nvidia.com:443",
function_id: str = "1598d209-5e27-4d3c-8079-4751568b1081",
+ sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
**kwargs,
):
- super().__init__(**kwargs)
+ super().__init__(sample_rate=sample_rate, **kwargs)
self._api_key = api_key
self._profanity_filter = False
self._automatic_punctuation = False
@@ -154,7 +154,6 @@ class ParakeetSTTService(STTService):
self._stop_history_eou = -1
self._stop_threshold_eou = -1.0
self._custom_configuration = ""
- self._sample_rate: int = 16000
self.set_model_name("parakeet-ctc-1.1b-asr")
@@ -166,6 +165,14 @@ class ParakeetSTTService(STTService):
self._asr_service = riva.client.ASRService(auth)
+ self._queue = asyncio.Queue()
+
+ def can_generate_metrics(self) -> bool:
+ return False
+
+ async def start(self, frame: StartFrame):
+ await super().start(frame)
+
config = riva.client.StreamingRecognitionConfig(
config=riva.client.RecognitionConfig(
encoding=riva.client.AudioEncoding.LINEAR_PCM,
@@ -175,14 +182,16 @@ class ParakeetSTTService(STTService):
profanity_filter=self._profanity_filter,
enable_automatic_punctuation=self._automatic_punctuation,
verbatim_transcripts=not self._no_verbatim_transcripts,
- sample_rate_hertz=self._sample_rate,
+ sample_rate_hertz=self.sample_rate,
audio_channel_count=1,
),
interim_results=True,
)
+
riva.client.add_word_boosting_to_config(
config, self._boosted_lm_words, self._boosted_lm_score
)
+
riva.client.add_endpoint_parameters_to_config(
config,
self._start_history,
@@ -193,17 +202,11 @@ class ParakeetSTTService(STTService):
self._stop_threshold_eou,
)
riva.client.add_custom_configuration_to_config(config, self._custom_configuration)
+
self._config = config
- self._queue = asyncio.Queue()
-
- def can_generate_metrics(self) -> bool:
- return False
-
- async def start(self, frame: StartFrame):
- await super().start(frame)
- self._thread_task = self.get_event_loop().create_task(self._thread_task_handler())
- self._response_task = self.get_event_loop().create_task(self._response_task_handler())
+ self._thread_task = self.create_task(self._thread_task_handler())
+ self._response_task = self.create_task(self._response_task_handler())
self._response_queue = asyncio.Queue()
async def stop(self, frame: EndFrame):
@@ -215,10 +218,8 @@ class ParakeetSTTService(STTService):
await self._stop_tasks()
async def _stop_tasks(self):
- self._thread_task.cancel()
- await self._thread_task
- self._response_task.cancel()
- await self._response_task
+ await self.cancel_task(self._thread_task)
+ await self.cancel_task(self._response_task)
def _response_handler(self):
responses = self._asr_service.streaming_response_generator(
@@ -238,7 +239,7 @@ class ParakeetSTTService(STTService):
await asyncio.to_thread(self._response_handler)
except asyncio.CancelledError:
self._thread_running = False
- pass
+ raise
async def _handle_response(self, response):
for result in response.results:
@@ -260,11 +261,8 @@ class ParakeetSTTService(STTService):
async def _response_task_handler(self):
while True:
- try:
- response = await self._response_queue.get()
- await self._handle_response(response)
- except asyncio.CancelledError:
- break
+ response = await self._response_queue.get()
+ await self._handle_response(response)
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
await self._queue.put(audio)
diff --git a/src/pipecat/services/simli.py b/src/pipecat/services/simli.py
index be7edfdbd..5a6aecbe4 100644
--- a/src/pipecat/services/simli.py
+++ b/src/pipecat/services/simli.py
@@ -49,45 +49,33 @@ class SimliVideoService(FrameProcessor):
async def _start_connection(self):
await self._simli_client.Initialize()
# Create task to consume and process audio and video
- self._audio_task = asyncio.create_task(self._consume_and_process_audio())
- self._video_task = asyncio.create_task(self._consume_and_process_video())
+ self._audio_task = self.create_task(self._consume_and_process_audio())
+ self._video_task = self.create_task(self._consume_and_process_video())
async def _consume_and_process_audio(self):
- try:
- await self._pipecat_resampler_event.wait()
- async for audio_frame in self._simli_client.getAudioStreamIterator():
- resampled_frames = self._pipecat_resampler.resample(audio_frame)
- for resampled_frame in resampled_frames:
- await self.push_frame(
- TTSAudioRawFrame(
- audio=resampled_frame.to_ndarray().tobytes(),
- sample_rate=self._pipecat_resampler.rate,
- num_channels=1,
- ),
- )
- except Exception as e:
- logger.exception(f"{self} exception: {e}")
- except asyncio.CancelledError:
- pass
+ await self._pipecat_resampler_event.wait()
+ async for audio_frame in self._simli_client.getAudioStreamIterator():
+ resampled_frames = self._pipecat_resampler.resample(audio_frame)
+ for resampled_frame in resampled_frames:
+ await self.push_frame(
+ TTSAudioRawFrame(
+ audio=resampled_frame.to_ndarray().tobytes(),
+ sample_rate=self._pipecat_resampler.rate,
+ num_channels=1,
+ ),
+ )
async def _consume_and_process_video(self):
- try:
- await self._pipecat_resampler_event.wait()
- async for video_frame in self._simli_client.getVideoStreamIterator(
- targetFormat="rgb24"
- ):
- # Process the video frame
- convertedFrame: OutputImageRawFrame = OutputImageRawFrame(
- image=video_frame.to_rgb().to_image().tobytes(),
- size=(video_frame.width, video_frame.height),
- format="RGB",
- )
- convertedFrame.pts = video_frame.pts
- await self.push_frame(convertedFrame)
- except Exception as e:
- logger.exception(f"{self} exception: {e}")
- except asyncio.CancelledError:
- pass
+ await self._pipecat_resampler_event.wait()
+ async for video_frame in self._simli_client.getVideoStreamIterator(targetFormat="rgb24"):
+ # Process the video frame
+ convertedFrame: OutputImageRawFrame = OutputImageRawFrame(
+ image=video_frame.to_rgb().to_image().tobytes(),
+ size=(video_frame.width, video_frame.height),
+ format="RGB",
+ )
+ convertedFrame.pts = video_frame.pts
+ await self.push_frame(convertedFrame)
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
@@ -128,8 +116,6 @@ class SimliVideoService(FrameProcessor):
async def _stop(self):
await self._simli_client.stop()
if self._audio_task:
- self._audio_task.cancel()
- await self._audio_task
+ await self.cancel_task(self._audio_task)
if self._video_task:
- self._video_task.cancel()
- await self._video_task
+ await self.cancel_task(self._video_task)
diff --git a/src/pipecat/services/tavus.py b/src/pipecat/services/tavus.py
index 7b43f6176..b0ca699cb 100644
--- a/src/pipecat/services/tavus.py
+++ b/src/pipecat/services/tavus.py
@@ -12,7 +12,7 @@ import base64
import aiohttp
from loguru import logger
-from pipecat.audio.utils import resample_audio
+from pipecat.audio.utils import create_default_resampler
from pipecat.frames.frames import (
CancelFrame,
EndFrame,
@@ -47,6 +47,8 @@ class TavusVideoService(AIService):
self._conversation_id: str
+ self._resampler = create_default_resampler()
+
async def initialize(self) -> str:
url = "https://tavusapi.com/v2/conversations"
headers = {"Content-Type": "application/json", "x-api-key": self._api_key}
@@ -75,18 +77,24 @@ class TavusVideoService(AIService):
logger.debug(f"TavusVideoService persona grabbed {response_json}")
return response_json["persona_name"]
+ async def stop(self, frame: EndFrame):
+ await super().stop(frame)
+ await self._end_conversation()
+
+ async def cancel(self, frame: CancelFrame):
+ await super().cancel(frame)
+ await self._end_conversation()
+
async def _end_conversation(self) -> None:
url = f"https://tavusapi.com/v2/conversations/{self._conversation_id}/end"
headers = {"Content-Type": "application/json", "x-api-key": self._api_key}
async with self._session.post(url, headers=headers) as r:
r.raise_for_status()
- async def _encode_audio_and_send(
- self, audio: bytes, original_sample_rate: int, done: bool
- ) -> None:
+ async def _encode_audio_and_send(self, audio: bytes, in_rate: int, done: bool) -> None:
"""Encodes audio to base64 and sends it to Tavus"""
if not done:
- audio = resample_audio(audio, original_sample_rate, 16000)
+ audio = await self._resampler.resample(audio, in_rate, 16000)
audio_base64 = base64.b64encode(audio).decode("utf-8")
logger.trace(f"{self}: sending {len(audio)} bytes")
await self._send_audio_message(audio_base64, done=done)
@@ -105,8 +113,6 @@ class TavusVideoService(AIService):
await self.stop_processing_metrics()
elif isinstance(frame, StartInterruptionFrame):
await self._send_interrupt_message()
- elif isinstance(frame, (EndFrame, CancelFrame)):
- await self._end_conversation()
else:
await self.push_frame(frame, direction)
diff --git a/src/pipecat/services/websocket_service.py b/src/pipecat/services/websocket_service.py
index 365f5a7c8..7e480f2b7 100644
--- a/src/pipecat/services/websocket_service.py
+++ b/src/pipecat/services/websocket_service.py
@@ -85,10 +85,6 @@ class WebsocketService(ABC):
await self._receive_messages()
logger.debug(f"{self} connection established successfully")
retry_count = 0 # Reset counter on successful message receive
-
- except asyncio.CancelledError:
- break
-
except Exception as e:
retry_count += 1
if retry_count >= MAX_RETRIES:
diff --git a/src/pipecat/services/xtts.py b/src/pipecat/services/xtts.py
index ba8803f51..7259b91d8 100644
--- a/src/pipecat/services/xtts.py
+++ b/src/pipecat/services/xtts.py
@@ -4,12 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
-from typing import Any, AsyncGenerator, Dict
+from typing import Any, AsyncGenerator, Dict, Optional
import aiohttp
from loguru import logger
-from pipecat.audio.utils import resample_audio
+from pipecat.audio.utils import create_default_resampler
from pipecat.frames.frames import (
ErrorFrame,
Frame,
@@ -76,7 +76,7 @@ class XTTSService(TTSService):
base_url: str,
aiohttp_session: aiohttp.ClientSession,
language: Language = Language.EN,
- sample_rate: int = 24000,
+ sample_rate: Optional[int] = None,
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
@@ -89,6 +89,8 @@ class XTTSService(TTSService):
self._studio_speakers: Dict[str, Any] | None = None
self._aiohttp_session = aiohttp_session
+ self._resampler = create_default_resampler()
+
def can_generate_metrics(self) -> bool:
return True
@@ -161,17 +163,19 @@ class XTTSService(TTSService):
buffer = buffer[48000:]
# XTTS uses 24000 so we need to resample to our desired rate.
- resampled_audio = resample_audio(
- bytes(process_data), 24000, self._sample_rate
+ resampled_audio = await self._resampler.resample(
+ bytes(process_data), 24000, self.sample_rate
)
# Create the frame with the resampled audio
- frame = TTSAudioRawFrame(resampled_audio, self._sample_rate, 1)
+ frame = TTSAudioRawFrame(resampled_audio, self.sample_rate, 1)
yield frame
# Process any remaining data in the buffer.
if len(buffer) > 0:
- resampled_audio = resample_audio(bytes(buffer), 24000, self._sample_rate)
- frame = TTSAudioRawFrame(resampled_audio, self._sample_rate, 1)
+ resampled_audio = await self._resampler.resample(
+ bytes(buffer), 24000, self.sample_rate
+ )
+ frame = TTSAudioRawFrame(resampled_audio, self.sample_rate, 1)
yield frame
yield TTSStoppedFrame()
diff --git a/src/pipecat/tests/__init__.py b/src/pipecat/tests/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/pipecat/tests/utils.py b/src/pipecat/tests/utils.py
new file mode 100644
index 000000000..ccfca9304
--- /dev/null
+++ b/src/pipecat/tests/utils.py
@@ -0,0 +1,128 @@
+#
+# Copyright (c) 2024-2025 Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import asyncio
+import sys
+from typing import Any, Awaitable, Callable, Dict, Sequence, Tuple
+
+from loguru import logger
+
+from pipecat.frames.frames import (
+ EndFrame,
+ Frame,
+ HeartbeatFrame,
+ StartFrame,
+)
+from pipecat.observers.base_observer import BaseObserver
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+
+logger.remove(0)
+logger.add(sys.stderr, level="TRACE")
+
+
+class HeartbeatsObserver(BaseObserver):
+ def __init__(
+ self,
+ *,
+ target: FrameProcessor,
+ heartbeat_callback: Callable[[FrameProcessor, HeartbeatFrame], Awaitable[None]],
+ ):
+ self._target = target
+ self._callback = heartbeat_callback
+
+ async def on_push_frame(
+ self,
+ src: FrameProcessor,
+ dst: FrameProcessor,
+ frame: Frame,
+ direction: FrameDirection,
+ timestamp: int,
+ ):
+ if src == self._target and isinstance(frame, HeartbeatFrame):
+ await self._callback(self._target, frame)
+
+
+class QueuedFrameProcessor(FrameProcessor):
+ def __init__(
+ self, queue: asyncio.Queue, queue_direction: FrameDirection, ignore_start: bool = True
+ ):
+ super().__init__()
+ self._queue = queue
+ self._queue_direction = queue_direction
+ self._ignore_start = ignore_start
+
+ async def process_frame(self, frame: Frame, direction: FrameDirection):
+ await super().process_frame(frame, direction)
+
+ if direction == self._queue_direction:
+ if not isinstance(frame, StartFrame) or not self._ignore_start:
+ await self._queue.put(frame)
+ await self.push_frame(frame, direction)
+
+
+async def run_test(
+ processor: FrameProcessor,
+ *,
+ frames_to_send: Sequence[Frame],
+ expected_down_frames: Sequence[type],
+ expected_up_frames: Sequence[type] = [],
+ ignore_start: bool = True,
+ start_metadata: Dict[str, Any] = {},
+ send_end_frame: bool = True,
+) -> Tuple[Sequence[Frame], Sequence[Frame]]:
+ received_up = asyncio.Queue()
+ received_down = asyncio.Queue()
+ source = QueuedFrameProcessor(received_up, FrameDirection.UPSTREAM, ignore_start)
+ sink = QueuedFrameProcessor(received_down, FrameDirection.DOWNSTREAM, ignore_start)
+
+ pipeline = Pipeline([source, processor, sink])
+
+ task = PipelineTask(pipeline, params=PipelineParams(start_metadata=start_metadata))
+
+ for frame in frames_to_send:
+ await task.queue_frame(frame)
+
+ if send_end_frame:
+ await task.queue_frame(EndFrame())
+
+ runner = PipelineRunner()
+ await runner.run(task)
+
+ #
+ # Down frames
+ #
+ received_down_frames: Sequence[Frame] = []
+ while not received_down.empty():
+ frame = await received_down.get()
+ if not isinstance(frame, EndFrame) or not send_end_frame:
+ received_down_frames.append(frame)
+
+ print("received DOWN frames =", received_down_frames)
+
+ assert len(received_down_frames) == len(expected_down_frames)
+
+ for real, expected in zip(received_down_frames, expected_down_frames):
+ assert isinstance(real, expected)
+
+ #
+ # Up frames
+ #
+ received_up_frames: Sequence[Frame] = []
+ while not received_up.empty():
+ frame = await received_up.get()
+ received_up_frames.append(frame)
+
+ print("received UP frames =", received_up_frames)
+
+ assert len(received_up_frames) == len(expected_up_frames)
+
+ for real, expected in zip(received_up_frames, expected_up_frames):
+ assert isinstance(real, expected)
+
+ return (received_down_frames, received_up_frames)
diff --git a/src/pipecat/transports/base_input.py b/src/pipecat/transports/base_input.py
index 93442f90a..b18776fa8 100644
--- a/src/pipecat/transports/base_input.py
+++ b/src/pipecat/transports/base_input.py
@@ -35,6 +35,9 @@ class BaseInputTransport(FrameProcessor):
self._params = params
+ # Input sample rate. It will be initialized on StartFrame.
+ self._sample_rate = 0
+
# We read audio from a single queue one at a time and we then run VAD in
# a thread. Therefore, only one thread should be necessary.
self._executor = ThreadPoolExecutor(max_workers=1)
@@ -43,20 +46,32 @@ class BaseInputTransport(FrameProcessor):
# if passthrough is enabled.
self._audio_task = None
+ @property
+ def sample_rate(self) -> int:
+ return self._sample_rate
+
+ @property
+ def vad_analyzer(self) -> VADAnalyzer | None:
+ return self._params.vad_analyzer
+
async def start(self, frame: StartFrame):
+ self._sample_rate = self._params.audio_in_sample_rate or frame.audio_in_sample_rate
+
+ # Configure VAD analyzer.
+ if self._params.vad_enabled and self._params.vad_analyzer:
+ self._params.vad_analyzer.set_sample_rate(self._sample_rate)
# Start audio filter.
if self._params.audio_in_filter:
- await self._params.audio_in_filter.start(self._params.audio_in_sample_rate)
+ await self._params.audio_in_filter.start(self._sample_rate)
# Create audio input queue and task if needed.
if self._params.audio_in_enabled or self._params.vad_enabled:
self._audio_in_queue = asyncio.Queue()
- self._audio_task = self.get_event_loop().create_task(self._audio_task_handler())
+ self._audio_task = self.create_task(self._audio_task_handler())
async def stop(self, frame: EndFrame):
# Cancel and wait for the audio input task to finish.
if self._audio_task and (self._params.audio_in_enabled or self._params.vad_enabled):
- self._audio_task.cancel()
- await self._audio_task
+ await self.cancel_task(self._audio_task)
self._audio_task = None
# Stop audio filter.
if self._params.audio_in_filter:
@@ -65,13 +80,9 @@ class BaseInputTransport(FrameProcessor):
async def cancel(self, frame: CancelFrame):
# Cancel and wait for the audio input task to finish.
if self._audio_task and (self._params.audio_in_enabled or self._params.vad_enabled):
- self._audio_task.cancel()
- await self._audio_task
+ await self.cancel_task(self._audio_task)
self._audio_task = None
- def vad_analyzer(self) -> VADAnalyzer | None:
- return self._params.vad_analyzer
-
async def push_audio_frame(self, frame: InputAudioRawFrame):
if self._params.audio_in_enabled or self._params.vad_enabled:
await self._audio_in_queue.put(frame)
@@ -106,9 +117,8 @@ class BaseInputTransport(FrameProcessor):
await self.push_frame(frame, direction)
await self.stop(frame)
elif isinstance(frame, VADParamsUpdateFrame):
- vad_analyzer = self.vad_analyzer()
- if vad_analyzer:
- vad_analyzer.set_params(frame.params)
+ if self.vad_analyzer:
+ self.vad_analyzer.set_params(frame.params)
elif isinstance(frame, FilterUpdateSettingsFrame) and self._params.audio_in_filter:
await self._params.audio_in_filter.process_frame(frame)
# Other frames
@@ -142,11 +152,10 @@ class BaseInputTransport(FrameProcessor):
async def _vad_analyze(self, audio_frame: InputAudioRawFrame) -> VADState:
state = VADState.QUIET
- vad_analyzer = self.vad_analyzer()
- if vad_analyzer:
+ if self.vad_analyzer:
logger.trace(f"{self}: analyzing VAD on {audio_frame}")
state = await self.get_event_loop().run_in_executor(
- self._executor, vad_analyzer.analyze_audio, audio_frame.audio
+ self._executor, self.vad_analyzer.analyze_audio, audio_frame.audio
)
logger.trace(f"{self}: done analyzing VAD on {audio_frame}")
return state
@@ -173,27 +182,22 @@ class BaseInputTransport(FrameProcessor):
async def _audio_task_handler(self):
vad_state: VADState = VADState.QUIET
while True:
- try:
- frame: InputAudioRawFrame = await self._audio_in_queue.get()
+ frame: InputAudioRawFrame = await self._audio_in_queue.get()
- audio_passthrough = True
+ audio_passthrough = True
- # If an audio filter is available, run it before VAD.
- if self._params.audio_in_filter:
- frame.audio = await self._params.audio_in_filter.filter(frame.audio)
+ # If an audio filter is available, run it before VAD.
+ if self._params.audio_in_filter:
+ frame.audio = await self._params.audio_in_filter.filter(frame.audio)
- # Check VAD and push event if necessary. We just care about
- # changes from QUIET to SPEAKING and vice versa.
- if self._params.vad_enabled:
- vad_state = await self._handle_vad(frame, vad_state)
- audio_passthrough = self._params.vad_audio_passthrough
+ # Check VAD and push event if necessary. We just care about
+ # changes from QUIET to SPEAKING and vice versa.
+ if self._params.vad_enabled:
+ vad_state = await self._handle_vad(frame, vad_state)
+ audio_passthrough = self._params.vad_audio_passthrough
- # Push audio downstream if passthrough.
- if audio_passthrough:
- await self.push_frame(frame)
+ # Push audio downstream if passthrough.
+ if audio_passthrough:
+ await self.push_frame(frame)
- self._audio_in_queue.task_done()
- except asyncio.CancelledError:
- break
- except Exception as e:
- logger.exception(f"{self} error reading audio frames: {e}")
+ self._audio_in_queue.task_done()
diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py
index ef073e22a..7a819b41c 100644
--- a/src/pipecat/transports/base_output.py
+++ b/src/pipecat/transports/base_output.py
@@ -57,12 +57,11 @@ class BaseOutputTransport(FrameProcessor):
# framerate.
self._camera_images = None
- # We will write 20ms audio at a time. If we receive long audio frames we
- # will chunk them. This will help with interruption handling.
- audio_bytes_10ms = (
- int(self._params.audio_out_sample_rate / 100) * self._params.audio_out_channels * 2
- )
- self._audio_chunk_size = audio_bytes_10ms * 2
+ # Output sample rate. It will be initialized on StartFrame.
+ self._sample_rate = 0
+
+ # Chunk size that will be written. It will be computed on StartFrame
+ self._audio_chunk_size = 0
self._audio_buffer = bytearray()
self._stopped_event = asyncio.Event()
@@ -70,10 +69,21 @@ class BaseOutputTransport(FrameProcessor):
# Indicates if the bot is currently speaking.
self._bot_speaking = False
+ @property
+ def sample_rate(self) -> int:
+ return self._sample_rate
+
async def start(self, frame: StartFrame):
+ self._sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
+
+ # We will write 20ms audio at a time. If we receive long audio frames we
+ # will chunk them. This will help with interruption handling.
+ audio_bytes_10ms = int(self._sample_rate / 100) * self._params.audio_out_channels * 2
+ self._audio_chunk_size = audio_bytes_10ms * 2
+
# Start audio mixer.
if self._params.audio_out_mixer:
- await self._params.audio_out_mixer.start(self._params.audio_out_sample_rate)
+ await self._params.audio_out_mixer.start(self._sample_rate)
self._create_camera_task()
self._create_sink_tasks()
@@ -87,9 +97,9 @@ class BaseOutputTransport(FrameProcessor):
# for these tasks before cancelling the camera and audio tasks below
# because they might be still rendering.
if self._sink_task:
- await self._sink_task
+ await self.wait_for_task(self._sink_task)
if self._sink_clock_task:
- await self._sink_clock_task
+ await self.wait_for_task(self._sink_clock_task)
# We can now cancel the camera task.
await self._cancel_camera_task()
@@ -217,22 +227,19 @@ class BaseOutputTransport(FrameProcessor):
#
def _create_sink_tasks(self):
- loop = self.get_event_loop()
self._sink_queue = asyncio.Queue()
- self._sink_task = loop.create_task(self._sink_task_handler())
self._sink_clock_queue = asyncio.PriorityQueue()
- self._sink_clock_task = loop.create_task(self._sink_clock_task_handler())
+ self._sink_task = self.create_task(self._sink_task_handler())
+ self._sink_clock_task = self.create_task(self._sink_clock_task_handler())
async def _cancel_sink_tasks(self):
# Stop sink tasks.
if self._sink_task:
- self._sink_task.cancel()
- await self._sink_task
+ await self.cancel_task(self._sink_task)
self._sink_task = None
# Stop sink clock tasks.
if self._sink_clock_task:
- self._sink_clock_task.cancel()
- await self._sink_clock_task
+ await self.cancel_task(self._sink_clock_task)
self._sink_clock_task = None
async def _sink_frame_handler(self, frame: Frame):
@@ -269,7 +276,7 @@ class BaseOutputTransport(FrameProcessor):
self._sink_clock_queue.task_done()
except asyncio.CancelledError:
- break
+ raise
except Exception as e:
logger.exception(f"{self} error processing sink clock queue: {e}")
@@ -301,7 +308,7 @@ class BaseOutputTransport(FrameProcessor):
# Generate an audio frame with only the mixer's part.
frame = OutputAudioRawFrame(
audio=await self._params.audio_out_mixer.mix(silence),
- sample_rate=self._params.audio_out_sample_rate,
+ sample_rate=self._sample_rate,
num_channels=self._params.audio_out_channels,
)
yield frame
@@ -317,49 +324,42 @@ class BaseOutputTransport(FrameProcessor):
return without_mixer(vad_stop_secs)
async def _sink_task_handler(self):
- try:
- async for frame in self._next_frame():
- # Notify the bot started speaking upstream if necessary and that
- # it's actually speaking.
- if isinstance(frame, TTSAudioRawFrame):
- await self._bot_started_speaking()
- await self.push_frame(BotSpeakingFrame())
- await self.push_frame(BotSpeakingFrame(), FrameDirection.UPSTREAM)
+ async for frame in self._next_frame():
+ # Notify the bot started speaking upstream if necessary and that
+ # it's actually speaking.
+ if isinstance(frame, TTSAudioRawFrame):
+ await self._bot_started_speaking()
+ await self.push_frame(BotSpeakingFrame())
+ await self.push_frame(BotSpeakingFrame(), FrameDirection.UPSTREAM)
- # No need to push EndFrame, it's pushed from process_frame().
- if isinstance(frame, EndFrame):
- break
+ # No need to push EndFrame, it's pushed from process_frame().
+ if isinstance(frame, EndFrame):
+ break
- # Handle frame.
- await self._sink_frame_handler(frame)
+ # Handle frame.
+ await self._sink_frame_handler(frame)
- # Also, push frame downstream in case anyone else needs it.
- await self.push_frame(frame)
+ # Also, push frame downstream in case anyone else needs it.
+ await self.push_frame(frame)
- # Send audio.
- if isinstance(frame, OutputAudioRawFrame):
- await self.write_raw_audio_frames(frame.audio)
- except asyncio.CancelledError:
- pass
- except Exception as e:
- logger.exception(f"{self} error writing to microphone: {e}")
+ # Send audio.
+ if isinstance(frame, OutputAudioRawFrame):
+ await self.write_raw_audio_frames(frame.audio)
#
# Camera task
#
def _create_camera_task(self):
- loop = self.get_event_loop()
# Create camera output queue and task if needed.
if self._params.camera_out_enabled:
self._camera_out_queue = asyncio.Queue()
- self._camera_out_task = loop.create_task(self._camera_out_task_handler())
+ self._camera_out_task = self.create_task(self._camera_out_task_handler())
async def _cancel_camera_task(self):
# Stop camera output task.
if self._camera_out_task and self._params.camera_out_enabled:
- self._camera_out_task.cancel()
- await self._camera_out_task
+ await self.cancel_task(self._camera_out_task)
self._camera_out_task = None
async def _draw_image(self, frame: OutputImageRawFrame):
@@ -387,19 +387,14 @@ class BaseOutputTransport(FrameProcessor):
self._camera_out_frame_duration = 1 / self._params.camera_out_framerate
self._camera_out_frame_reset = self._camera_out_frame_duration * 5
while True:
- try:
- if self._params.camera_out_is_live:
- await self._camera_out_is_live_handler()
- elif self._camera_images:
- image = next(self._camera_images)
- await self._draw_image(image)
- await asyncio.sleep(self._camera_out_frame_duration)
- else:
- await asyncio.sleep(self._camera_out_frame_duration)
- except asyncio.CancelledError:
- break
- except Exception as e:
- logger.exception(f"{self} error writing to camera: {e}")
+ if self._params.camera_out_is_live:
+ await self._camera_out_is_live_handler()
+ elif self._camera_images:
+ image = next(self._camera_images)
+ await self._draw_image(image)
+ await asyncio.sleep(self._camera_out_frame_duration)
+ else:
+ await asyncio.sleep(self._camera_out_frame_duration)
async def _camera_out_is_live_handler(self):
image = await self._camera_out_queue.get()
diff --git a/src/pipecat/transports/base_transport.py b/src/pipecat/transports/base_transport.py
index bcc13c4fd..f9c1eb65e 100644
--- a/src/pipecat/transports/base_transport.py
+++ b/src/pipecat/transports/base_transport.py
@@ -16,6 +16,7 @@ from pipecat.audio.filters.base_audio_filter import BaseAudioFilter
from pipecat.audio.mixers.base_audio_mixer import BaseAudioMixer
from pipecat.audio.vad.vad_analyzer import VADAnalyzer
from pipecat.processors.frame_processor import FrameProcessor
+from pipecat.utils.utils import obj_count, obj_id
class TransportParams(BaseModel):
@@ -30,12 +31,12 @@ class TransportParams(BaseModel):
camera_out_color_format: str = "RGB"
audio_out_enabled: bool = False
audio_out_is_live: bool = False
- audio_out_sample_rate: int = 24000
+ audio_out_sample_rate: Optional[int] = None
audio_out_channels: int = 1
audio_out_bitrate: int = 96000
audio_out_mixer: Optional[BaseAudioMixer] = None
audio_in_enabled: bool = False
- audio_in_sample_rate: int = 16000
+ audio_in_sample_rate: Optional[int] = None
audio_in_channels: int = 1
audio_in_filter: Optional[BaseAudioFilter] = None
vad_enabled: bool = False
@@ -46,15 +47,25 @@ class TransportParams(BaseModel):
class BaseTransport(ABC):
def __init__(
self,
- input_name: str | None = None,
- output_name: str | None = None,
- loop: asyncio.AbstractEventLoop | None = None,
+ *,
+ name: Optional[str] = None,
+ input_name: Optional[str] = None,
+ output_name: Optional[str] = None,
):
+ self._id: int = obj_id()
+ self._name = name or f"{self.__class__.__name__}#{obj_count(self)}"
self._input_name = input_name
self._output_name = output_name
- self._loop = loop or asyncio.get_running_loop()
self._event_handlers: dict = {}
+ @property
+ def id(self) -> int:
+ return self._id
+
+ @property
+ def name(self) -> str:
+ return self._name
+
@abstractmethod
def input(self) -> FrameProcessor:
raise NotImplementedError
@@ -89,3 +100,6 @@ class BaseTransport(ABC):
handler(self, *args, **kwargs)
except Exception as e:
logger.exception(f"Exception in event handler {event_name}: {e}")
+
+ def __str__(self):
+ return self.name
diff --git a/src/pipecat/transports/local/audio.py b/src/pipecat/transports/local/audio.py
index c65279be9..946ce5bda 100644
--- a/src/pipecat/transports/local/audio.py
+++ b/src/pipecat/transports/local/audio.py
@@ -28,35 +28,40 @@ except ModuleNotFoundError as e:
class LocalAudioInputTransport(BaseInputTransport):
def __init__(self, py_audio: pyaudio.PyAudio, params: TransportParams):
super().__init__(params)
+ self._py_audio = py_audio
+ self._in_stream = None
+ self._sample_rate = 0
- sample_rate = self._params.audio_in_sample_rate
- num_frames = int(sample_rate / 100) * 2 # 20ms of audio
+ async def start(self, frame: StartFrame):
+ await super().start(frame)
- self._in_stream = py_audio.open(
- format=py_audio.get_format_from_width(2),
- channels=params.audio_in_channels,
- rate=params.audio_in_sample_rate,
+ self._sample_rate = self._params.audio_in_sample_rate or frame.audio_in_sample_rate
+ num_frames = int(self._sample_rate / 100) * 2 # 20ms of audio
+
+ self._in_stream = self._py_audio.open(
+ format=self._py_audio.get_format_from_width(2),
+ channels=self._params.audio_in_channels,
+ rate=self._sample_rate,
frames_per_buffer=num_frames,
stream_callback=self._audio_in_callback,
input=True,
)
-
- async def start(self, frame: StartFrame):
- await super().start(frame)
self._in_stream.start_stream()
async def cleanup(self):
await super().cleanup()
- self._in_stream.stop_stream()
- # This is not very pretty (taken from PyAudio docs).
- while self._in_stream.is_active():
- await asyncio.sleep(0.1)
- self._in_stream.close()
+ if self._in_stream:
+ self._in_stream.stop_stream()
+ # This is not very pretty (taken from PyAudio docs).
+ while self._in_stream.is_active():
+ await asyncio.sleep(0.1)
+ self._in_stream.close()
+ self._in_stream = None
def _audio_in_callback(self, in_data, frame_count, time_info, status):
frame = InputAudioRawFrame(
audio=in_data,
- sample_rate=self._params.audio_in_sample_rate,
+ sample_rate=self._sample_rate,
num_channels=self._params.audio_in_channels,
)
@@ -68,36 +73,46 @@ class LocalAudioInputTransport(BaseInputTransport):
class LocalAudioOutputTransport(BaseOutputTransport):
def __init__(self, py_audio: pyaudio.PyAudio, params: TransportParams):
super().__init__(params)
+ self._py_audio = py_audio
+ self._out_stream = None
+ self._sample_rate = 0
# We only write audio frames from a single task, so only one thread
# should be necessary.
self._executor = ThreadPoolExecutor(max_workers=1)
- self._out_stream = py_audio.open(
- format=py_audio.get_format_from_width(2),
- channels=params.audio_out_channels,
- rate=params.audio_out_sample_rate,
- output=True,
- )
-
async def start(self, frame: StartFrame):
await super().start(frame)
+
+ self._sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
+
+ self._out_stream = self._py_audio.open(
+ format=self._py_audio.get_format_from_width(2),
+ channels=self._params.audio_out_channels,
+ rate=self._sample_rate,
+ output=True,
+ )
self._out_stream.start_stream()
async def cleanup(self):
await super().cleanup()
- self._out_stream.stop_stream()
- # This is not very pretty (taken from PyAudio docs).
- while self._out_stream.is_active():
- await asyncio.sleep(0.1)
- self._out_stream.close()
+ if self._out_stream:
+ self._out_stream.stop_stream()
+ # This is not very pretty (taken from PyAudio docs).
+ while self._out_stream.is_active():
+ await asyncio.sleep(0.1)
+ self._out_stream.close()
async def write_raw_audio_frames(self, frames: bytes):
- await self.get_event_loop().run_in_executor(self._executor, self._out_stream.write, frames)
+ if self._out_stream:
+ await self.get_event_loop().run_in_executor(
+ self._executor, self._out_stream.write, frames
+ )
class LocalAudioTransport(BaseTransport):
def __init__(self, params: TransportParams):
+ super().__init__()
self._params = params
self._pyaudio = pyaudio.PyAudio()
diff --git a/src/pipecat/transports/local/tk.py b/src/pipecat/transports/local/tk.py
index 842eac2df..7d6fa6c75 100644
--- a/src/pipecat/transports/local/tk.py
+++ b/src/pipecat/transports/local/tk.py
@@ -36,35 +36,39 @@ except ModuleNotFoundError as e:
class TkInputTransport(BaseInputTransport):
def __init__(self, py_audio: pyaudio.PyAudio, params: TransportParams):
super().__init__(params)
+ self._py_audio = py_audio
+ self._in_stream = None
+ self._sample_rate = 0
- sample_rate = self._params.audio_in_sample_rate
- num_frames = int(sample_rate / 100) * 2 # 20ms of audio
+ async def start(self, frame: StartFrame):
+ await super().start(frame)
- self._in_stream = py_audio.open(
- format=py_audio.get_format_from_width(2),
- channels=params.audio_in_channels,
- rate=params.audio_in_sample_rate,
+ self._sample_rate = self._params.audio_in_sample_rate or frame.audio_in_sample_rate
+ num_frames = int(self._sample_rate / 100) * 2 # 20ms of audio
+
+ self._in_stream = self._py_audio.open(
+ format=self._py_audio.get_format_from_width(2),
+ channels=self._params.audio_in_channels,
+ rate=self._sample_rate,
frames_per_buffer=num_frames,
stream_callback=self._audio_in_callback,
input=True,
)
-
- async def start(self, frame: StartFrame):
- await super().start(frame)
self._in_stream.start_stream()
async def cleanup(self):
await super().cleanup()
- self._in_stream.stop_stream()
- # This is not very pretty (taken from PyAudio docs).
- while self._in_stream.is_active():
- await asyncio.sleep(0.1)
- self._in_stream.close()
+ if self._in_stream:
+ self._in_stream.stop_stream()
+ # This is not very pretty (taken from PyAudio docs).
+ while self._in_stream.is_active():
+ await asyncio.sleep(0.1)
+ self._in_stream.close()
def _audio_in_callback(self, in_data, frame_count, time_info, status):
frame = InputAudioRawFrame(
audio=in_data,
- sample_rate=self._params.audio_in_sample_rate,
+ sample_rate=self._sample_rate,
num_channels=self._params.audio_in_channels,
)
@@ -76,18 +80,14 @@ class TkInputTransport(BaseInputTransport):
class TkOutputTransport(BaseOutputTransport):
def __init__(self, tk_root: tk.Tk, py_audio: pyaudio.PyAudio, params: TransportParams):
super().__init__(params)
+ self._py_audio = py_audio
+ self._out_stream = None
+ self._sample_rate = 0
# We only write audio frames from a single task, so only one thread
# should be necessary.
self._executor = ThreadPoolExecutor(max_workers=1)
- self._out_stream = py_audio.open(
- format=py_audio.get_format_from_width(2),
- channels=params.audio_out_channels,
- rate=params.audio_out_sample_rate,
- output=True,
- )
-
# Start with a neutral gray background.
array = np.ones((1024, 1024, 3)) * 128
data = f"P5 {1024} {1024} 255 ".encode() + array.astype(np.uint8).tobytes()
@@ -97,18 +97,31 @@ class TkOutputTransport(BaseOutputTransport):
async def start(self, frame: StartFrame):
await super().start(frame)
+
+ self._sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
+
+ self._out_stream = self._py_audio.open(
+ format=self._py_audio.get_format_from_width(2),
+ channels=self._params.audio_out_channels,
+ rate=self._sample_rate,
+ output=True,
+ )
self._out_stream.start_stream()
async def cleanup(self):
await super().cleanup()
- self._out_stream.stop_stream()
- # This is not very pretty (taken from PyAudio docs).
- while self._out_stream.is_active():
- await asyncio.sleep(0.1)
- self._out_stream.close()
+ if self._out_stream:
+ self._out_stream.stop_stream()
+ # This is not very pretty (taken from PyAudio docs).
+ while self._out_stream.is_active():
+ await asyncio.sleep(0.1)
+ self._out_stream.close()
async def write_raw_audio_frames(self, frames: bytes):
- await self.get_event_loop().run_in_executor(self._executor, self._out_stream.write, frames)
+ if self._out_stream:
+ await self.get_event_loop().run_in_executor(
+ self._executor, self._out_stream.write, frames
+ )
async def write_frame_to_camera(self, frame: OutputImageRawFrame):
self.get_event_loop().call_soon(self._write_frame_to_tk, frame)
@@ -127,6 +140,7 @@ class TkOutputTransport(BaseOutputTransport):
class TkLocalTransport(BaseTransport):
def __init__(self, tk_root: tk.Tk, params: TransportParams):
+ super().__init__()
self._tk_root = tk_root
self._params = params
self._pyaudio = pyaudio.PyAudio()
diff --git a/src/pipecat/transports/network/fastapi_websocket.py b/src/pipecat/transports/network/fastapi_websocket.py
index fc5804611..106bcb30d 100644
--- a/src/pipecat/transports/network/fastapi_websocket.py
+++ b/src/pipecat/transports/network/fastapi_websocket.py
@@ -16,6 +16,8 @@ from loguru import logger
from pydantic import BaseModel
from pipecat.frames.frames import (
+ CancelFrame,
+ EndFrame,
Frame,
InputAudioRawFrame,
OutputAudioRawFrame,
@@ -67,12 +69,19 @@ class FastAPIWebsocketInputTransport(BaseInputTransport):
async def start(self, frame: StartFrame):
await super().start(frame)
+ await self._params.serializer.setup(frame)
if self._params.session_timeout:
- self._monitor_websocket_task = self.get_event_loop().create_task(
- self._monitor_websocket()
- )
+ self._monitor_websocket_task = self.create_task(self._monitor_websocket())
await self._callbacks.on_client_connected(self._websocket)
- self._receive_task = self.get_event_loop().create_task(self._receive_messages())
+ self._receive_task = self.create_task(self._receive_messages())
+
+ async def stop(self, frame: EndFrame):
+ await super().stop(frame)
+ await self.cancel_task(self._receive_task)
+
+ async def cancel(self, frame: CancelFrame):
+ await super().cancel(frame)
+ await self.cancel_task(self._receive_task)
def _iter_data(self) -> typing.AsyncIterator[bytes | str]:
if self._params.serializer.type == FrameSerializerType.BINARY:
@@ -81,26 +90,26 @@ class FastAPIWebsocketInputTransport(BaseInputTransport):
return self._websocket.iter_text()
async def _receive_messages(self):
- async for message in self._iter_data():
- frame = self._params.serializer.deserialize(message)
+ try:
+ async for message in self._iter_data():
+ frame = await self._params.serializer.deserialize(message)
- if not frame:
- continue
+ if not frame:
+ continue
- if isinstance(frame, InputAudioRawFrame):
- await self.push_audio_frame(frame)
- else:
- await self.push_frame(frame)
+ if isinstance(frame, InputAudioRawFrame):
+ await self.push_audio_frame(frame)
+ else:
+ await self.push_frame(frame)
+ except Exception as e:
+ logger.error(f"{self} exception receiving data: {e.__class__.__name__} ({e})")
await self._callbacks.on_client_disconnected(self._websocket)
async def _monitor_websocket(self):
"""Wait for self._params.session_timeout seconds, if the websocket is still open, trigger timeout event."""
- try:
- await asyncio.sleep(self._params.session_timeout)
- await self._callbacks.on_session_timeout(self._websocket)
- except asyncio.CancelledError:
- logger.info(f"Monitoring task cancelled for: {self._websocket}")
+ await asyncio.sleep(self._params.session_timeout)
+ await self._callbacks.on_session_timeout(self._websocket)
class FastAPIWebsocketOutputTransport(BaseOutputTransport):
@@ -110,9 +119,19 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
self._websocket = websocket
self._params = params
- self._send_interval = (self._audio_chunk_size / self._params.audio_out_sample_rate) / 2
+ # write_raw_audio_frames() is called quickly, as soon as we get audio
+ # (e.g. from the TTS), and since this is just a network connection we
+ # would be sending it to quickly. Instead, we want to block to emulate
+ # an audio device, this is what the send interval is. It will be
+ # computed on StartFrame.
+ self._send_interval = 0
self._next_send_time = 0
+ async def start(self, frame: StartFrame):
+ await super().start(frame)
+ await self._params.serializer.setup(frame)
+ self._send_interval = (self._audio_chunk_size / self.sample_rate) / 2
+
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
@@ -128,7 +147,7 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
frame = OutputAudioRawFrame(
audio=frames,
- sample_rate=self._params.audio_out_sample_rate,
+ sample_rate=self.sample_rate,
num_channels=self._params.audio_out_channels,
)
@@ -154,9 +173,12 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
await self._write_audio_sleep()
async def _write_frame(self, frame: Frame):
- payload = self._params.serializer.serialize(frame)
- if payload and self._websocket.client_state == WebSocketState.CONNECTED:
- await self._send_data(payload)
+ try:
+ payload = await self._params.serializer.serialize(frame)
+ if payload and self._websocket.client_state == WebSocketState.CONNECTED:
+ await self._send_data(payload)
+ except Exception as e:
+ logger.error(f"{self} exception sending data: {e.__class__.__name__} ({e})")
def _send_data(self, data: str | bytes):
if self._params.serializer.type == FrameSerializerType.BINARY:
@@ -182,9 +204,8 @@ class FastAPIWebsocketTransport(BaseTransport):
params: FastAPIWebsocketParams,
input_name: str | None = None,
output_name: str | None = None,
- loop: asyncio.AbstractEventLoop | None = None,
):
- super().__init__(input_name=input_name, output_name=output_name, loop=loop)
+ super().__init__(input_name=input_name, output_name=output_name)
self._params = params
self._callbacks = FastAPIWebsocketCallbacks(
diff --git a/src/pipecat/transports/network/websocket_client.py b/src/pipecat/transports/network/websocket_client.py
new file mode 100644
index 000000000..2de4e2914
--- /dev/null
+++ b/src/pipecat/transports/network/websocket_client.py
@@ -0,0 +1,269 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+
+import asyncio
+import io
+import time
+import wave
+from typing import Awaitable, Callable, Optional
+
+import websockets
+from loguru import logger
+from pydantic.main import BaseModel
+
+from pipecat.frames.frames import (
+ CancelFrame,
+ EndFrame,
+ Frame,
+ InputAudioRawFrame,
+ OutputAudioRawFrame,
+ StartFrame,
+ TransportMessageFrame,
+ TransportMessageUrgentFrame,
+)
+from pipecat.serializers.base_serializer import FrameSerializer
+from pipecat.serializers.protobuf import ProtobufFrameSerializer
+from pipecat.transports.base_input import BaseInputTransport
+from pipecat.transports.base_output import BaseOutputTransport
+from pipecat.transports.base_transport import BaseTransport, TransportParams
+from pipecat.utils.asyncio import TaskManager
+
+
+class WebsocketClientParams(TransportParams):
+ add_wav_header: bool = True
+ serializer: FrameSerializer = ProtobufFrameSerializer()
+
+
+class WebsocketClientCallbacks(BaseModel):
+ on_connected: Callable[[websockets.WebSocketClientProtocol], Awaitable[None]]
+ on_disconnected: Callable[[websockets.WebSocketClientProtocol], Awaitable[None]]
+ on_message: Callable[[websockets.WebSocketClientProtocol, websockets.Data], Awaitable[None]]
+
+
+class WebsocketClientSession:
+ def __init__(
+ self,
+ uri: str,
+ params: WebsocketClientParams,
+ callbacks: WebsocketClientCallbacks,
+ transport_name: str,
+ ):
+ self._uri = uri
+ self._params = params
+ self._callbacks = callbacks
+ self._transport_name = transport_name
+
+ self._task_manager: Optional[TaskManager] = None
+
+ self._websocket: websockets.WebSocketClientProtocol | None = None
+
+ @property
+ def task_manager(self) -> TaskManager:
+ if not self._task_manager:
+ raise Exception(
+ f"{self._transport_name}::WebsocketClientSession: TaskManager not initialized (pipeline not started?)"
+ )
+ return self._task_manager
+
+ async def setup(self, frame: StartFrame):
+ if not self._task_manager:
+ self._task_manager = frame.task_manager
+
+ async def connect(self):
+ if self._websocket:
+ return
+
+ try:
+ self._websocket = await websockets.connect(uri=self._uri, open_timeout=10)
+ self._client_task = self.task_manager.create_task(
+ self._client_task_handler(),
+ f"{self._transport_name}::WebsocketClientSession::_client_task_handler",
+ )
+ await self._callbacks.on_connected(self._websocket)
+ except TimeoutError:
+ logger.error(f"Timeout connecting to {self._uri}")
+
+ async def disconnect(self):
+ if not self._websocket:
+ return
+
+ await self.task_manager.cancel_task(self._client_task)
+
+ await self._websocket.close()
+ self._websocket = None
+
+ async def send(self, message: websockets.Data):
+ try:
+ if self._websocket:
+ await self._websocket.send(message)
+ except Exception as e:
+ logger.error(f"{self} exception sending data: {e.__class__.__name__} ({e})")
+
+ async def _client_task_handler(self):
+ try:
+ # Handle incoming messages
+ async for message in self._websocket:
+ await self._callbacks.on_message(self._websocket, message)
+ except Exception as e:
+ logger.error(f"{self} exception receiving data: {e.__class__.__name__} ({e})")
+
+ await self._callbacks.on_disconnected(self._websocket)
+
+ def __str__(self):
+ return f"{self._transport_name}::WebsocketClientSession"
+
+
+class WebsocketClientInputTransport(BaseInputTransport):
+ def __init__(self, session: WebsocketClientSession, params: WebsocketClientParams):
+ super().__init__(params)
+
+ self._session = session
+ self._params = params
+
+ async def start(self, frame: StartFrame):
+ await super().start(frame)
+ await self._params.serializer.setup(frame)
+ await self._session.setup(frame)
+ await self._session.connect()
+
+ async def stop(self, frame: EndFrame):
+ await super().stop(frame)
+ await self._session.disconnect()
+
+ async def cancel(self, frame: CancelFrame):
+ await super().cancel(frame)
+ await self._session.disconnect()
+
+ async def on_message(self, websocket, message):
+ frame = await self._params.serializer.deserialize(message)
+ if not frame:
+ return
+ if isinstance(frame, InputAudioRawFrame) and self._params.audio_in_enabled:
+ await self.push_audio_frame(frame)
+ else:
+ await self.push_frame(frame)
+
+
+class WebsocketClientOutputTransport(BaseOutputTransport):
+ def __init__(self, session: WebsocketClientSession, params: WebsocketClientParams):
+ super().__init__(params)
+
+ self._session = session
+ self._params = params
+
+ # write_raw_audio_frames() is called quickly, as soon as we get audio
+ # (e.g. from the TTS), and since this is just a network connection we
+ # would be sending it to quickly. Instead, we want to block to emulate
+ # an audio device, this is what the send interval is. It will be
+ # computed on StartFrame.
+ self._send_interval = 0
+ self._next_send_time = 0
+
+ async def start(self, frame: StartFrame):
+ await super().start(frame)
+ self._send_interval = (self._audio_chunk_size / self.sample_rate) / 2
+ await self._params.serializer.setup(frame)
+ await self._session.setup(frame)
+ await self._session.connect()
+
+ async def stop(self, frame: EndFrame):
+ await super().stop(frame)
+ await self._session.disconnect()
+
+ async def cancel(self, frame: CancelFrame):
+ await super().cancel(frame)
+ await self._session.disconnect()
+
+ async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
+ await self._write_frame(frame)
+
+ async def write_raw_audio_frames(self, frames: bytes):
+ frame = OutputAudioRawFrame(
+ audio=frames,
+ sample_rate=self.sample_rate,
+ num_channels=self._params.audio_out_channels,
+ )
+
+ if self._params.add_wav_header:
+ with io.BytesIO() as buffer:
+ with wave.open(buffer, "wb") as wf:
+ wf.setsampwidth(2)
+ wf.setnchannels(frame.num_channels)
+ wf.setframerate(frame.sample_rate)
+ wf.writeframes(frame.audio)
+ wav_frame = OutputAudioRawFrame(
+ buffer.getvalue(),
+ sample_rate=frame.sample_rate,
+ num_channels=frame.num_channels,
+ )
+ frame = wav_frame
+
+ await self._write_frame(frame)
+
+ # Simulate audio playback with a sleep.
+ await self._write_audio_sleep()
+
+ async def _write_frame(self, frame: Frame):
+ payload = await self._params.serializer.serialize(frame)
+ if payload:
+ await self._session.send(payload)
+
+ async def _write_audio_sleep(self):
+ # Simulate a clock.
+ current_time = time.monotonic()
+ sleep_duration = max(0, self._next_send_time - current_time)
+ await asyncio.sleep(sleep_duration)
+ if sleep_duration == 0:
+ self._next_send_time = time.monotonic() + self._send_interval
+ else:
+ self._next_send_time += self._send_interval
+
+
+class WebsocketClientTransport(BaseTransport):
+ def __init__(
+ self,
+ uri: str,
+ params: WebsocketClientParams = WebsocketClientParams(),
+ ):
+ super().__init__()
+
+ self._params = params
+
+ callbacks = WebsocketClientCallbacks(
+ on_connected=self._on_connected,
+ on_disconnected=self._on_disconnected,
+ on_message=self._on_message,
+ )
+
+ self._session = WebsocketClientSession(uri, params, callbacks, self.name)
+ self._input: WebsocketClientInputTransport | None = None
+ self._output: WebsocketClientOutputTransport | None = None
+
+ # Register supported handlers. The user will only be able to register
+ # these handlers.
+ self._register_event_handler("on_connected")
+ self._register_event_handler("on_disconnected")
+
+ def input(self) -> WebsocketClientInputTransport:
+ if not self._input:
+ self._input = WebsocketClientInputTransport(self._session, self._params)
+ return self._input
+
+ def output(self) -> WebsocketClientOutputTransport:
+ if not self._output:
+ self._output = WebsocketClientOutputTransport(self._session, self._params)
+ return self._output
+
+ async def _on_connected(self, websocket):
+ await self._call_event_handler("on_connected", websocket)
+
+ async def _on_disconnected(self, websocket):
+ await self._call_event_handler("on_disconnected", websocket)
+
+ async def _on_message(self, websocket, message):
+ if self._input:
+ await self._input.on_message(websocket, message)
diff --git a/src/pipecat/transports/network/websocket_server.py b/src/pipecat/transports/network/websocket_server.py
index 9bd344cb1..1d8420116 100644
--- a/src/pipecat/transports/network/websocket_server.py
+++ b/src/pipecat/transports/network/websocket_server.py
@@ -24,7 +24,6 @@ from pipecat.frames.frames import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.serializers.base_serializer import FrameSerializer
-from pipecat.serializers.protobuf import ProtobufFrameSerializer
from pipecat.transports.base_input import BaseInputTransport
from pipecat.transports.base_output import BaseOutputTransport
from pipecat.transports.base_transport import BaseTransport, TransportParams
@@ -39,7 +38,7 @@ except ModuleNotFoundError as e:
class WebsocketServerParams(TransportParams):
add_wav_header: bool = False
- serializer: FrameSerializer = ProtobufFrameSerializer()
+ serializer: FrameSerializer
session_timeout: int | None = None
@@ -67,21 +66,32 @@ class WebsocketServerInputTransport(BaseInputTransport):
self._websocket: websockets.WebSocketServerProtocol | None = None
+ self._server_task = None
+
+ # This task will monitor the websocket connection periodically.
+ self._monitor_task = None
+
self._stop_server_event = asyncio.Event()
async def start(self, frame: StartFrame):
await super().start(frame)
- self._server_task = self.get_event_loop().create_task(self._server_task_handler())
+ await self._params.serializer.setup(frame)
+ self._server_task = self.create_task(self._server_task_handler())
async def stop(self, frame: EndFrame):
await super().stop(frame)
self._stop_server_event.set()
- await self._server_task
+ if self._monitor_task:
+ await self.cancel_task(self._monitor_task)
+ if self._server_task:
+ await self.wait_for_task(self._server_task)
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
- self._stop_server_event.set()
- await self._server_task
+ if self._monitor_task:
+ await self.cancel_task(self._monitor_task)
+ if self._server_task:
+ await self.cancel_task(self._server_task)
async def _server_task_handler(self):
logger.info(f"Starting websocket server on {self._host}:{self._port}")
@@ -101,19 +111,24 @@ class WebsocketServerInputTransport(BaseInputTransport):
# Create a task to monitor the websocket connection
if self._params.session_timeout:
- self.get_event_loop().create_task(self._monitor_websocket(websocket))
+ self._monitor_task = self.create_task(
+ self._monitor_websocket(websocket, self._params.session_timeout)
+ )
# Handle incoming messages
- async for message in websocket:
- frame = self._params.serializer.deserialize(message)
+ try:
+ async for message in websocket:
+ frame = await self._params.serializer.deserialize(message)
- if not frame:
- continue
+ if not frame:
+ continue
- if isinstance(frame, InputAudioRawFrame):
- await self.push_audio_frame(frame)
- else:
- await self.push_frame(frame)
+ if isinstance(frame, InputAudioRawFrame):
+ await self.push_audio_frame(frame)
+ else:
+ await self.push_frame(frame)
+ except Exception as e:
+ logger.error(f"{self} exception receiving data: {e.__class__.__name__} ({e})")
# Notify disconnection
await self._callbacks.on_client_disconnected(websocket)
@@ -123,14 +138,18 @@ class WebsocketServerInputTransport(BaseInputTransport):
logger.info(f"Client {websocket.remote_address} disconnected")
- async def _monitor_websocket(self, websocket: websockets.WebSocketServerProtocol):
- """Wait for self._params.session_timeout seconds, if the websocket is still open, trigger timeout event."""
+ async def _monitor_websocket(
+ self, websocket: websockets.WebSocketServerProtocol, session_timeout: int
+ ):
+ """Wait for session_timeout seconds, if the websocket is still open,
+ trigger timeout event."""
try:
- await asyncio.sleep(self._params.session_timeout)
+ await asyncio.sleep(session_timeout)
if not websocket.closed:
await self._callbacks.on_session_timeout(websocket)
except asyncio.CancelledError:
logger.info(f"Monitoring task cancelled for: {websocket.remote_address}")
+ raise
class WebsocketServerOutputTransport(BaseOutputTransport):
@@ -141,9 +160,12 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
self._websocket: websockets.WebSocketServerProtocol | None = None
- self._websocket_audio_buffer = bytes()
-
- self._send_interval = (self._audio_chunk_size / self._params.audio_out_sample_rate) / 2
+ # write_raw_audio_frames() is called quickly, as soon as we get audio
+ # (e.g. from the TTS), and since this is just a network connection we
+ # would be sending it to quickly. Instead, we want to block to emulate
+ # an audio device, this is what the send interval is. It will be
+ # computed on StartFrame.
+ self._send_interval = 0
self._next_send_time = 0
async def set_client_connection(self, websocket: websockets.WebSocketServerProtocol | None):
@@ -152,6 +174,11 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
logger.warning("Only one client allowed, using new connection")
self._websocket = websocket
+ async def start(self, frame: StartFrame):
+ await super().start(frame)
+ await self._params.serializer.setup(frame)
+ self._send_interval = (self._audio_chunk_size / self.sample_rate) / 2
+
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
@@ -167,7 +194,7 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
frame = OutputAudioRawFrame(
audio=frames,
- sample_rate=self._params.audio_out_sample_rate,
+ sample_rate=self.sample_rate,
num_channels=self._params.audio_out_channels,
)
@@ -187,15 +214,16 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
await self._write_frame(frame)
- self._websocket_audio_buffer = bytes()
-
# Simulate audio playback with a sleep.
await self._write_audio_sleep()
async def _write_frame(self, frame: Frame):
- payload = self._params.serializer.serialize(frame)
- if payload and self._websocket:
- await self._websocket.send(payload)
+ try:
+ payload = await self._params.serializer.serialize(frame)
+ if payload and self._websocket:
+ await self._websocket.send(payload)
+ except Exception as e:
+ logger.error(f"{self} exception sending data: {e.__class__.__name__} ({e})")
async def _write_audio_sleep(self):
# Simulate a clock.
@@ -211,14 +239,13 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
class WebsocketServerTransport(BaseTransport):
def __init__(
self,
+ params: WebsocketServerParams,
host: str = "localhost",
port: int = 8765,
- params: WebsocketServerParams = WebsocketServerParams(),
input_name: str | None = None,
output_name: str | None = None,
- loop: asyncio.AbstractEventLoop | None = None,
):
- super().__init__(input_name=input_name, output_name=output_name, loop=loop)
+ super().__init__(input_name=input_name, output_name=output_name)
self._host = host
self._port = port
self._params = params
diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py
index 47575bcce..13e52df2c 100644
--- a/src/pipecat/transports/services/daily.py
+++ b/src/pipecat/transports/services/daily.py
@@ -46,6 +46,7 @@ from pipecat.transcriptions.language import Language
from pipecat.transports.base_input import BaseInputTransport
from pipecat.transports.base_output import BaseOutputTransport
from pipecat.transports.base_transport import BaseTransport, TransportParams
+from pipecat.utils.asyncio import TaskManager
try:
from daily import CallClient, Daily, EventHandler
@@ -70,11 +71,11 @@ class DailyTransportMessageUrgentFrame(TransportMessageUrgentFrame):
class WebRTCVADAnalyzer(VADAnalyzer):
- def __init__(self, *, sample_rate=16000, num_channels=1, params: VADParams = VADParams()):
- super().__init__(sample_rate=sample_rate, num_channels=num_channels, params=params)
+ def __init__(self, *, sample_rate: Optional[int] = None, params: VADParams = VADParams()):
+ super().__init__(sample_rate=sample_rate, params=params)
self._webrtc_vad = Daily.create_native_vad(
- reset_period_ms=VAD_RESET_PERIOD_MS, sample_rate=sample_rate, channels=num_channels
+ reset_period_ms=VAD_RESET_PERIOD_MS, sample_rate=self.sample_rate, channels=1
)
logger.debug("Loaded native WebRTC VAD")
@@ -139,7 +140,6 @@ class DailyCallbacks(BaseModel):
on_dialout_stopped: Callable[[Any], Awaitable[None]]
on_dialout_error: Callable[[Any], Awaitable[None]]
on_dialout_warning: Callable[[Any], Awaitable[None]]
- on_first_participant_joined: Callable[[Mapping[str, Any]], Awaitable[None]]
on_participant_joined: Callable[[Mapping[str, Any]], Awaitable[None]]
on_participant_left: Callable[[Mapping[str, Any], str], Awaitable[None]]
on_participant_updated: Callable[[Mapping[str, Any]], Awaitable[None]]
@@ -179,7 +179,7 @@ class DailyTransportClient(EventHandler):
bot_name: str,
params: DailyParams,
callbacks: DailyCallbacks,
- loop: asyncio.AbstractEventLoop,
+ transport_name: str,
):
super().__init__()
@@ -192,17 +192,19 @@ class DailyTransportClient(EventHandler):
self._bot_name: str = bot_name
self._params: DailyParams = params
self._callbacks = callbacks
- self._loop = loop
+ self._transport_name = transport_name
self._participant_id: str = ""
self._video_renderers = {}
self._transcription_ids = []
self._transcription_status = None
- self._other_participant_has_joined = False
self._joined = False
+ self._joined_event = asyncio.Event()
self._leave_counter = 0
+ self._task_manager: Optional[TaskManager] = None
+
# We use the executor to cleanup the client. We just do it from one
# place, so only one thread is really needed.
self._executor = ThreadPoolExecutor(max_workers=1)
@@ -218,35 +220,15 @@ class DailyTransportClient(EventHandler):
# future) we will deadlock because completions use event handlers (which
# are holding the GIL).
self._callback_queue = asyncio.Queue()
- self._callback_task = self._loop.create_task(self._callback_task_handler())
+ self._callback_task = None
+
+ # Input and ouput sample rates. They will be initialize on setup().
+ self._in_sample_rate = 0
+ self._out_sample_rate = 0
self._camera: VirtualCameraDevice | None = None
- if self._params.camera_out_enabled:
- self._camera = Daily.create_camera_device(
- self._camera_name(),
- width=self._params.camera_out_width,
- height=self._params.camera_out_height,
- color_format=self._params.camera_out_color_format,
- )
-
self._mic: VirtualMicrophoneDevice | None = None
- if self._params.audio_out_enabled:
- self._mic = Daily.create_microphone_device(
- self._mic_name(),
- sample_rate=self._params.audio_out_sample_rate,
- channels=self._params.audio_out_channels,
- non_blocking=True,
- )
-
self._speaker: VirtualSpeakerDevice | None = None
- if self._params.audio_in_enabled or self._params.vad_enabled:
- self._speaker = Daily.create_speaker_device(
- self._speaker_name(),
- sample_rate=self._params.audio_in_sample_rate,
- channels=self._params.audio_in_channels,
- non_blocking=True,
- )
- Daily.select_speaker_device(self._speaker_name())
def _camera_name(self):
return f"camera-{self}"
@@ -261,9 +243,6 @@ class DailyTransportClient(EventHandler):
def participant_id(self) -> str:
return self._participant_id
- def set_callbacks(self, callbacks: DailyCallbacks):
- self._callbacks = callbacks
-
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
if not self._joined:
return
@@ -272,7 +251,7 @@ class DailyTransportClient(EventHandler):
if isinstance(frame, (DailyTransportMessageFrame, DailyTransportMessageUrgentFrame)):
participant_id = frame.participant_id
- future = self._loop.create_future()
+ future = self._get_event_loop().create_future()
self._client.send_app_message(
frame.message, participant_id, completion=completion_callback(future)
)
@@ -282,11 +261,11 @@ class DailyTransportClient(EventHandler):
if not self._speaker:
return None
- sample_rate = self._params.audio_in_sample_rate
+ sample_rate = self._in_sample_rate
num_channels = self._params.audio_in_channels
num_frames = int(sample_rate / 100) * 2 # 20ms of audio
- future = self._loop.create_future()
+ future = self._get_event_loop().create_future()
self._speaker.read_frames(num_frames, completion=completion_callback(future))
audio = await future
@@ -305,7 +284,7 @@ class DailyTransportClient(EventHandler):
if not self._mic:
return None
- future = self._loop.create_future()
+ future = self._get_event_loop().create_future()
self._mic.write_frames(frames, completion=completion_callback(future))
await future
@@ -315,6 +294,42 @@ class DailyTransportClient(EventHandler):
self._camera.write_frame(frame.image)
+ async def setup(self, frame: StartFrame):
+ self._in_sample_rate = self._params.audio_in_sample_rate or frame.audio_in_sample_rate
+ self._out_sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
+
+ if self._params.camera_out_enabled and not self._camera:
+ self._camera = Daily.create_camera_device(
+ self._camera_name(),
+ width=self._params.camera_out_width,
+ height=self._params.camera_out_height,
+ color_format=self._params.camera_out_color_format,
+ )
+
+ if self._params.audio_out_enabled and not self._mic:
+ self._mic = Daily.create_microphone_device(
+ self._mic_name(),
+ sample_rate=self._out_sample_rate,
+ channels=self._params.audio_out_channels,
+ non_blocking=True,
+ )
+
+ if (self._params.audio_in_enabled or self._params.vad_enabled) and not self._speaker:
+ self._speaker = Daily.create_speaker_device(
+ self._speaker_name(),
+ sample_rate=self._in_sample_rate,
+ channels=self._params.audio_in_channels,
+ non_blocking=True,
+ )
+ Daily.select_speaker_device(self._speaker_name())
+
+ if not self._task_manager:
+ self._task_manager = frame.task_manager
+ self._callback_task = self._task_manager.create_task(
+ self._callback_task_handler(),
+ f"{self}::callback_task",
+ )
+
async def join(self):
# Transport already joined, ignore.
if self._joined:
@@ -346,6 +361,8 @@ class DailyTransportClient(EventHandler):
await self._start_transcription()
await self._callbacks.on_joined(data)
+
+ self._joined_event.set()
else:
error_msg = f"Error joining {self._room_url}: {error}"
logger.error(error_msg)
@@ -362,7 +379,7 @@ class DailyTransportClient(EventHandler):
logger.info(f"Enabling transcription with settings {self._params.transcription_settings}")
- future = self._loop.create_future()
+ future = self._get_event_loop().create_future()
self._client.start_transcription(
settings=self._params.transcription_settings.model_dump(exclude_none=True),
completion=completion_callback(future),
@@ -373,7 +390,7 @@ class DailyTransportClient(EventHandler):
return
async def _join(self):
- future = self._loop.create_future()
+ future = self._get_event_loop().create_future()
self._client.join(
self._room_url,
@@ -434,6 +451,7 @@ class DailyTransportClient(EventHandler):
return
self._joined = False
+ self._joined_event.clear()
logger.info(f"Leaving {self._room_url}")
@@ -457,23 +475,24 @@ class DailyTransportClient(EventHandler):
async def _stop_transcription(self):
if not self._token:
return
- future = self._loop.create_future()
+ future = self._get_event_loop().create_future()
self._client.stop_transcription(completion=completion_callback(future))
error = await future
if error:
logger.error(f"Unable to stop transcription: {error}")
async def _leave(self):
- future = self._loop.create_future()
+ future = self._get_event_loop().create_future()
self._client.leave(completion=completion_callback(future))
return await asyncio.wait_for(future, timeout=10)
async def cleanup(self):
- self._callback_task.cancel()
- await self._callback_task
+ if self._callback_task and self._task_manager:
+ await self._task_manager.cancel_task(self._callback_task)
+ self._callback_task = None
# Make sure we don't block the event loop in case `client.release()`
# takes extra time.
- await self._loop.run_in_executor(self._executor, self._cleanup)
+ await self._get_event_loop().run_in_executor(self._executor, self._cleanup)
def _cleanup(self):
if self._client:
@@ -487,39 +506,39 @@ class DailyTransportClient(EventHandler):
return self._client.participant_counts()
async def start_dialout(self, settings):
- future = self._loop.create_future()
+ future = self._get_event_loop().create_future()
self._client.start_dialout(settings, completion=completion_callback(future))
await future
async def stop_dialout(self, participant_id):
- future = self._loop.create_future()
+ future = self._get_event_loop().create_future()
self._client.stop_dialout(participant_id, completion=completion_callback(future))
await future
async def send_dtmf(self, settings):
- future = self._loop.create_future()
+ future = self._get_event_loop().create_future()
self._client.send_dtmf(settings, completion=completion_callback(future))
await future
async def sip_call_transfer(self, settings):
- future = self._loop.create_future()
+ future = self._get_event_loop().create_future()
self._client.sip_call_transfer(settings, completion=completion_callback(future))
await future
async def sip_refer(self, settings):
- future = self._loop.create_future()
+ future = self._get_event_loop().create_future()
self._client.sip_refer(settings, completion=completion_callback(future))
await future
async def start_recording(self, streaming_settings, stream_id, force_new):
- future = self._loop.create_future()
+ future = self._get_event_loop().create_future()
self._client.start_recording(
streaming_settings, stream_id, force_new, completion=completion_callback(future)
)
await future
async def stop_recording(self, stream_id):
- future = self._loop.create_future()
+ future = self._get_event_loop().create_future()
self._client.stop_recording(stream_id, completion=completion_callback(future))
await future
@@ -527,7 +546,7 @@ class DailyTransportClient(EventHandler):
if not self._joined:
return
- future = self._loop.create_future()
+ future = self._get_event_loop().create_future()
self._client.send_prebuilt_chat_message(
message, user_name=user_name, completion=completion_callback(future)
)
@@ -564,14 +583,14 @@ class DailyTransportClient(EventHandler):
)
async def update_transcription(self, participants=None, instance_id=None):
- future = self._loop.create_future()
+ future = self._get_event_loop().create_future()
self._client.update_transcription(
participants, instance_id, completion=completion_callback(future)
)
await future
async def update_subscriptions(self, participant_settings=None, profile_settings=None):
- future = self._loop.create_future()
+ future = self._get_event_loop().create_future()
self._client.update_subscriptions(
participant_settings=participant_settings,
profile_settings=profile_settings,
@@ -621,19 +640,9 @@ class DailyTransportClient(EventHandler):
self._call_async_callback(self._callbacks.on_dialout_warning, data)
def on_participant_joined(self, participant):
- id = participant["id"]
- logger.info(f"Participant joined {id}")
-
- if not self._other_participant_has_joined:
- self._other_participant_has_joined = True
- self._call_async_callback(self._callbacks.on_first_participant_joined, participant)
-
self._call_async_callback(self._callbacks.on_participant_joined, participant)
def on_participant_left(self, participant, reason):
- id = participant["id"]
- logger.info(f"Participant left {id}")
-
self._call_async_callback(self._callbacks.on_participant_left, participant, reason)
def on_participant_updated(self, participant):
@@ -681,17 +690,24 @@ class DailyTransportClient(EventHandler):
def _call_async_callback(self, callback, *args):
future = asyncio.run_coroutine_threadsafe(
- self._callback_queue.put((callback, *args)), self._loop
+ self._callback_queue.put((callback, *args)), self._get_event_loop()
)
future.result()
async def _callback_task_handler(self):
while True:
- try:
- (callback, *args) = await self._callback_queue.get()
- await callback(*args)
- except asyncio.CancelledError:
- break
+ # Wait to process any callback until we are joined.
+ await self._joined_event.wait()
+ (callback, *args) = await self._callback_queue.get()
+ await callback(*args)
+
+ def _get_event_loop(self) -> asyncio.AbstractEventLoop:
+ if not self._task_manager:
+ raise Exception(f"{self}: missing task manager (pipeline not started?)")
+ return self._task_manager.get_event_loop()
+
+ def __str__(self):
+ return f"{self._transport_name}::DailyTransportClient"
class DailyInputTransport(BaseInputTransport):
@@ -699,6 +715,7 @@ class DailyInputTransport(BaseInputTransport):
super().__init__(params, **kwargs)
self._client = client
+ self._params = params
self._video_renderers = {}
@@ -707,21 +724,25 @@ class DailyInputTransport(BaseInputTransport):
self._audio_in_task = None
self._vad_analyzer: VADAnalyzer | None = params.vad_analyzer
- if params.vad_enabled and not params.vad_analyzer:
- self._vad_analyzer = WebRTCVADAnalyzer(
- sample_rate=self._params.audio_in_sample_rate,
- num_channels=self._params.audio_in_channels,
- )
+
+ @property
+ def vad_analyzer(self) -> VADAnalyzer | None:
+ return self._vad_analyzer
async def start(self, frame: StartFrame):
# Parent start.
await super().start(frame)
+ # Setup client.
+ await self._client.setup(frame)
# Join the room.
await self._client.join()
+ # Inialize WebRTC VAD if needed.
+ if self._params.vad_enabled and not self._params.vad_analyzer:
+ self._vad_analyzer = WebRTCVADAnalyzer(sample_rate=self.sample_rate)
# Create audio task. It reads audio frames from Daily and push them
# internally for VAD processing.
if self._params.audio_in_enabled or self._params.vad_enabled:
- self._audio_in_task = self.get_event_loop().create_task(self._audio_in_task_handler())
+ self._audio_in_task = self.create_task(self._audio_in_task_handler())
async def stop(self, frame: EndFrame):
# Parent stop.
@@ -730,8 +751,7 @@ class DailyInputTransport(BaseInputTransport):
await self._client.leave()
# Stop audio thread.
if self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled):
- self._audio_in_task.cancel()
- await self._audio_in_task
+ await self.cancel_task(self._audio_in_task)
self._audio_in_task = None
async def cancel(self, frame: CancelFrame):
@@ -741,17 +761,13 @@ class DailyInputTransport(BaseInputTransport):
await self._client.leave()
# Stop audio thread.
if self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled):
- self._audio_in_task.cancel()
- await self._audio_in_task
+ await self.cancel_task(self._audio_in_task)
self._audio_in_task = None
async def cleanup(self):
await super().cleanup()
await self._client.cleanup()
- def vad_analyzer(self) -> VADAnalyzer | None:
- return self._vad_analyzer
-
#
# FrameProcessor
#
@@ -779,12 +795,9 @@ class DailyInputTransport(BaseInputTransport):
async def _audio_in_task_handler(self):
while True:
- try:
- frame = await self._client.read_next_audio_frame()
- if frame:
- await self.push_audio_frame(frame)
- except asyncio.CancelledError:
- break
+ frame = await self._client.read_next_audio_frame()
+ if frame:
+ await self.push_audio_frame(frame)
#
# Camera in
@@ -843,6 +856,8 @@ class DailyOutputTransport(BaseOutputTransport):
async def start(self, frame: StartFrame):
# Parent start.
await super().start(frame)
+ # Setup client.
+ await self._client.setup(frame)
# Join the room.
await self._client.join()
@@ -881,9 +896,8 @@ class DailyTransport(BaseTransport):
params: DailyParams = DailyParams(),
input_name: str | None = None,
output_name: str | None = None,
- loop: asyncio.AbstractEventLoop | None = None,
):
- super().__init__(input_name=input_name, output_name=output_name, loop=loop)
+ super().__init__(input_name=input_name, output_name=output_name)
callbacks = DailyCallbacks(
on_joined=self._on_joined,
@@ -901,7 +915,6 @@ class DailyTransport(BaseTransport):
on_dialout_stopped=self._on_dialout_stopped,
on_dialout_error=self._on_dialout_error,
on_dialout_warning=self._on_dialout_warning,
- on_first_participant_joined=self._on_first_participant_joined,
on_participant_joined=self._on_participant_joined,
on_participant_left=self._on_participant_left,
on_participant_updated=self._on_participant_updated,
@@ -912,12 +925,12 @@ class DailyTransport(BaseTransport):
)
self._params = params
- self._client = DailyTransportClient(
- room_url, token, bot_name, params, callbacks, self._loop
- )
+ self._client = DailyTransportClient(room_url, token, bot_name, params, callbacks, self.name)
self._input: DailyInputTransport | None = None
self._output: DailyOutputTransport | None = None
+ self._other_participant_has_joined = False
+
# Register supported handlers. The user will only be able to register
# these handlers.
self._register_event_handler("on_joined")
@@ -1047,7 +1060,7 @@ class DailyTransport(BaseTransport):
await self._output.push_error(error_frame)
else:
logger.error("Both input and output are None while trying to push error")
- raise RuntimeError("No valid input or output channel to push error")
+ raise Exception("No valid input or output channel to push error")
async def _on_app_message(self, message: Any, sender: str):
if self._input:
@@ -1124,17 +1137,23 @@ class DailyTransport(BaseTransport):
await self._call_event_handler("on_dialout_warning", data)
async def _on_participant_joined(self, participant):
+ id = participant["id"]
+ logger.info(f"Participant joined {id}")
+
+ if not self._other_participant_has_joined:
+ self._other_participant_has_joined = True
+ await self._call_event_handler("on_first_participant_joined", participant)
+
await self._call_event_handler("on_participant_joined", participant)
async def _on_participant_left(self, participant, reason):
+ id = participant["id"]
+ logger.info(f"Participant left {id}")
await self._call_event_handler("on_participant_left", participant, reason)
async def _on_participant_updated(self, participant):
await self._call_event_handler("on_participant_updated", participant)
- async def _on_first_participant_joined(self, participant):
- await self._call_event_handler("on_first_participant_joined", participant)
-
async def _on_transcription_message(self, message):
await self._call_event_handler("on_transcription_message", message)
diff --git a/src/pipecat/transports/services/helpers/daily_rest.py b/src/pipecat/transports/services/helpers/daily_rest.py
index 65b6a1d62..9becacebc 100644
--- a/src/pipecat/transports/services/helpers/daily_rest.py
+++ b/src/pipecat/transports/services/helpers/daily_rest.py
@@ -33,6 +33,19 @@ class DailyRoomSipParams(BaseModel):
num_endpoints: int = 1
+class RecordingsBucketConfig(BaseModel):
+ """Configuration for storing Daily recordings in a custom S3 bucket.
+
+ Refer to the Daily API documentation for more information:
+ https://docs.daily.co/guides/products/live-streaming-recording/storing-recordings-in-a-custom-s3-bucket
+ """
+
+ bucket_name: str
+ bucket_region: str
+ assume_role_arn: str
+ allow_api_access: bool = False
+
+
class DailyRoomProperties(BaseModel, extra="allow"):
"""Properties for configuring a Daily room.
@@ -43,6 +56,8 @@ class DailyRoomProperties(BaseModel, extra="allow"):
enable_emoji_reactions: Whether emoji reactions are enabled
eject_at_room_exp: Whether to remove participants when room expires
enable_dialout: Whether SIP dial-out is enabled
+ enable_recording: Recording settings ('cloud', 'local', 'raw-tracks')
+ geo: Geographic region for room
max_participants: Maximum number of participants allowed in the room
sip: SIP configuration parameters
sip_uri: SIP URI information returned by Daily
@@ -57,7 +72,10 @@ class DailyRoomProperties(BaseModel, extra="allow"):
enable_emoji_reactions: bool = False
eject_at_room_exp: bool = True
enable_dialout: Optional[bool] = None
+ enable_recording: Optional[Literal["cloud", "local", "raw-tracks"]] = None
+ geo: Optional[str] = None
max_participants: Optional[int] = None
+ recordings_bucket: Optional[RecordingsBucketConfig] = None
sip: Optional[DailyRoomSipParams] = None
sip_uri: Optional[dict] = None
start_video_off: bool = False
@@ -111,6 +129,84 @@ class DailyRoomObject(BaseModel):
config: DailyRoomProperties
+class DailyMeetingTokenProperties(BaseModel):
+ """Properties for configuring a Daily meeting token.
+
+ Refer to the Daily API documentation for more information:
+ https://docs.daily.co/reference/rest-api/meeting-tokens/create-meeting-token#properties
+ """
+
+ room_name: Optional[str] = Field(
+ default=None,
+ description="The room for which this token is valid. If not set, the token is valid for all rooms in your domain. You should always set room_name if using this token to control meeting access.",
+ )
+
+ eject_at_token_exp: Optional[bool] = Field(
+ default=None,
+ description="If `true`, the user will be ejected from the room when the token expires. Defaults to `false`.",
+ )
+ eject_after_elapsed: Optional[int] = Field(
+ default=None,
+ description="The number of seconds after which the user will be ejected from the room. If not provided, the user will not be ejected based on elapsed time.",
+ )
+
+ nbf: Optional[int] = Field(
+ default=None,
+ description="Not before. This is a unix timestamp (seconds since the epoch.) Users cannot join a meeting in with this token before this time.",
+ )
+
+ exp: Optional[int] = Field(
+ default=None,
+ description="Expiration time (unix timestamp in seconds). We strongly recommend setting this value for security. If not set, the token will not expire. Refer docs for more info.",
+ )
+ is_owner: Optional[bool] = Field(
+ default=None,
+ description="If `true`, the token will grant owner privileges in the room. Defaults to `false`.",
+ )
+ user_name: Optional[str] = Field(
+ default=None,
+ description="The name of the user. This will be added to the token payload.",
+ )
+ user_id: Optional[str] = Field(
+ default=None,
+ description="A unique identifier for the user. This will be added to the token payload.",
+ )
+ enable_screenshare: Optional[bool] = Field(
+ default=None,
+ description="If `true`, the user will be able to share their screen. Defaults to `true`.",
+ )
+ start_video_off: Optional[bool] = Field(
+ default=None,
+ description="If `true`, the user's video will be turned off when they join the room. Defaults to `false`.",
+ )
+ start_audio_off: Optional[bool] = Field(
+ default=None,
+ description="If `true`, the user's audio will be turned off when they join the room. Defaults to `false`.",
+ )
+ enable_recording: Optional[Literal["cloud", "local", "raw-tracks"]] = Field(
+ default=None,
+ description="Recording settings for the token. Must be one of `cloud`, `local` or `raw-tracks`.",
+ )
+ enable_prejoin_ui: Optional[bool] = Field(
+ default=None,
+ description="If `true`, the user will see the prejoin UI before joining the room.",
+ )
+ start_cloud_recording: Optional[bool] = Field(
+ default=None,
+ description="Start cloud recording when the user joins the room. This can be used to always record and archive meetings, for example in a customer support context.",
+ )
+
+
+class DailyMeetingTokenParams(BaseModel):
+ """Parameters for creating a Daily meeting token.
+
+ Refer to the Daily API documentation for more information:
+ https://docs.daily.co/reference/rest-api/meeting-tokens/create-meeting-token#body-params
+ """
+
+ properties: DailyMeetingTokenProperties = Field(default_factory=DailyMeetingTokenProperties)
+
+
class DailyRESTHelper:
"""Helper class for interacting with Daily's REST API.
@@ -129,6 +225,7 @@ class DailyRESTHelper:
daily_api_url: str = "https://api.daily.co/v1",
aiohttp_session: aiohttp.ClientSession,
):
+ """Initialize the Daily REST helper."""
self.daily_api_key = daily_api_key
self.daily_api_url = daily_api_url
self.aiohttp_session = aiohttp_session
@@ -169,7 +266,7 @@ class DailyRESTHelper:
Exception: If room creation fails or response is invalid
"""
headers = {"Authorization": f"Bearer {self.daily_api_key}"}
- json = {**params.model_dump(exclude_none=True)}
+ json = params.model_dump(exclude_none=True)
async with self.aiohttp_session.post(
f"{self.daily_api_url}/rooms", headers=headers, json=json
) as r:
@@ -187,7 +284,11 @@ class DailyRESTHelper:
return room
async def get_token(
- self, room_url: str, expiry_time: float = 60 * 60, owner: bool = True
+ self,
+ room_url: str,
+ expiry_time: float = 60 * 60,
+ owner: bool = True,
+ params: Optional[DailyMeetingTokenParams] = None,
) -> str:
"""Generate a meeting token for user to join a Daily room.
@@ -195,6 +296,9 @@ class DailyRESTHelper:
room_url: Daily room URL
expiry_time: Token validity duration in seconds (default: 1 hour)
owner: Whether token has owner privileges
+ params: Optional additional token properties. Note that room_name,
+ exp, and is_owner will be set based on the other function
+ parameters regardless of values in params.
Returns:
str: Meeting token
@@ -207,12 +311,25 @@ class DailyRESTHelper:
"No Daily room specified. You must specify a Daily room in order a token to be generated."
)
- expiration: float = time.time() + expiry_time
+ expiration: int = int(time.time() + expiry_time)
room_name = self.get_name_from_url(room_url)
headers = {"Authorization": f"Bearer {self.daily_api_key}"}
- json = {"properties": {"room_name": room_name, "is_owner": owner, "exp": expiration}}
+
+ if params is None:
+ params = DailyMeetingTokenParams(
+ properties=DailyMeetingTokenProperties(
+ room_name=room_name, is_owner=owner, exp=expiration
+ )
+ )
+ else:
+ params.properties.room_name = room_name
+ params.properties.exp = expiration
+ params.properties.is_owner = owner
+
+ json = params.model_dump(exclude_none=True)
+
async with self.aiohttp_session.post(
f"{self.daily_api_url}/meeting-tokens", headers=headers, json=json
) as r:
diff --git a/src/pipecat/transports/services/livekit.py b/src/pipecat/transports/services/livekit.py
index 0b036cecb..925f7dae1 100644
--- a/src/pipecat/transports/services/livekit.py
+++ b/src/pipecat/transports/services/livekit.py
@@ -6,18 +6,17 @@
import asyncio
from dataclasses import dataclass
-from typing import Any, Awaitable, Callable, List
+from typing import Any, Awaitable, Callable, List, Optional
from loguru import logger
from pydantic import BaseModel
-from pipecat.audio.utils import resample_audio
+from pipecat.audio.utils import create_default_resampler
from pipecat.audio.vad.vad_analyzer import VADAnalyzer
from pipecat.frames.frames import (
AudioRawFrame,
CancelFrame,
EndFrame,
- Frame,
InputAudioRawFrame,
OutputAudioRawFrame,
StartFrame,
@@ -28,6 +27,7 @@ from pipecat.processors.frame_processor import FrameDirection
from pipecat.transports.base_input import BaseInputTransport
from pipecat.transports.base_output import BaseOutputTransport
from pipecat.transports.base_transport import BaseTransport, TransportParams
+from pipecat.utils.asyncio import TaskManager
try:
from livekit import rtc
@@ -71,15 +71,15 @@ class LiveKitTransportClient:
room_name: str,
params: LiveKitParams,
callbacks: LiveKitCallbacks,
- loop: asyncio.AbstractEventLoop,
+ transport_name: str,
):
self._url = url
self._token = token
self._room_name = room_name
self._params = params
self._callbacks = callbacks
- self._loop = loop
- self._room = rtc.Room(loop=loop)
+ self._transport_name = transport_name
+ self._room: rtc.Room | None = None
self._participant_id: str = ""
self._connected = False
self._disconnect_counter = 0
@@ -88,20 +88,33 @@ class LiveKitTransportClient:
self._audio_tracks = {}
self._audio_queue = asyncio.Queue()
self._other_participant_has_joined = False
-
- # Set up room event handlers
- self._room.on("participant_connected")(self._on_participant_connected_wrapper)
- self._room.on("participant_disconnected")(self._on_participant_disconnected_wrapper)
- self._room.on("track_subscribed")(self._on_track_subscribed_wrapper)
- self._room.on("track_unsubscribed")(self._on_track_unsubscribed_wrapper)
- self._room.on("data_received")(self._on_data_received_wrapper)
- self._room.on("connected")(self._on_connected_wrapper)
- self._room.on("disconnected")(self._on_disconnected_wrapper)
+ self._task_manager: Optional[TaskManager] = None
@property
def participant_id(self) -> str:
return self._participant_id
+ @property
+ def room(self) -> rtc.Room:
+ if not self._room:
+ raise Exception(f"{self}: missing room object (pipeline not started?)")
+ return self._room
+
+ async def setup(self, frame: StartFrame):
+ self._out_sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
+ if not self._task_manager:
+ self._task_manager = frame.task_manager
+ self._room = rtc.Room(loop=self._task_manager.get_event_loop())
+
+ # Set up room event handlers
+ self.room.on("participant_connected")(self._on_participant_connected_wrapper)
+ self.room.on("participant_disconnected")(self._on_participant_disconnected_wrapper)
+ self.room.on("track_subscribed")(self._on_track_subscribed_wrapper)
+ self.room.on("track_unsubscribed")(self._on_track_unsubscribed_wrapper)
+ self.room.on("data_received")(self._on_data_received_wrapper)
+ self.room.on("connected")(self._on_connected_wrapper)
+ self.room.on("disconnected")(self._on_disconnected_wrapper)
+
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
async def connect(self):
if self._connected:
@@ -112,7 +125,7 @@ class LiveKitTransportClient:
logger.info(f"Connecting to {self._room_name}")
try:
- await self._room.connect(
+ await self.room.connect(
self._url,
self._token,
options=rtc.RoomOptions(auto_subscribe=True),
@@ -121,19 +134,19 @@ class LiveKitTransportClient:
# Increment disconnect counter if we successfully connected.
self._disconnect_counter += 1
- self._participant_id = self._room.local_participant.sid
+ self._participant_id = self.room.local_participant.sid
logger.info(f"Connected to {self._room_name}")
# Set up audio source and track
self._audio_source = rtc.AudioSource(
- self._params.audio_out_sample_rate, self._params.audio_out_channels
+ self._out_sample_rate, self._params.audio_out_channels
)
self._audio_track = rtc.LocalAudioTrack.create_audio_track(
"pipecat-audio", self._audio_source
)
options = rtc.TrackPublishOptions()
options.source = rtc.TrackSource.SOURCE_MICROPHONE
- await self._room.local_participant.publish_track(self._audio_track, options)
+ await self.room.local_participant.publish_track(self._audio_track, options)
await self._callbacks.on_connected()
@@ -154,7 +167,7 @@ class LiveKitTransportClient:
return
logger.info(f"Disconnecting from {self._room_name}")
- await self._room.disconnect()
+ await self.room.disconnect()
self._connected = False
logger.info(f"Disconnected from {self._room_name}")
await self._callbacks.on_disconnected()
@@ -165,11 +178,11 @@ class LiveKitTransportClient:
try:
if participant_id:
- await self._room.local_participant.publish_data(
+ await self.room.local_participant.publish_data(
data, reliable=True, destination_identities=[participant_id]
)
else:
- await self._room.local_participant.publish_data(data, reliable=True)
+ await self.room.local_participant.publish_data(data, reliable=True)
except Exception as e:
logger.error(f"Error sending data: {e}")
@@ -183,10 +196,10 @@ class LiveKitTransportClient:
logger.error(f"Error publishing audio: {e}")
def get_participants(self) -> List[str]:
- return [p.sid for p in self._room.remote_participants.values()]
+ return [p.sid for p in self.room.remote_participants.values()]
async def get_participant_metadata(self, participant_id: str) -> dict:
- participant = self._room.remote_participants.get(participant_id)
+ participant = self.room.remote_participants.get(participant_id)
if participant:
return {
"id": participant.sid,
@@ -197,17 +210,17 @@ class LiveKitTransportClient:
return {}
async def set_participant_metadata(self, metadata: str):
- await self._room.local_participant.set_metadata(metadata)
+ await self.room.local_participant.set_metadata(metadata)
async def mute_participant(self, participant_id: str):
- participant = self._room.remote_participants.get(participant_id)
+ participant = self.room.remote_participants.get(participant_id)
if participant:
for track in participant.tracks.values():
if track.kind == "audio":
await track.set_enabled(False)
async def unmute_participant(self, participant_id: str):
- participant = self._room.remote_participants.get(participant_id)
+ participant = self.room.remote_participants.get(participant_id)
if participant:
for track in participant.tracks.values():
if track.kind == "audio":
@@ -215,10 +228,16 @@ class LiveKitTransportClient:
# Wrapper methods for event handlers
def _on_participant_connected_wrapper(self, participant: rtc.RemoteParticipant):
- asyncio.create_task(self._async_on_participant_connected(participant))
+ self._task_manager.create_task(
+ self._async_on_participant_connected(participant),
+ f"{self}::_async_on_participant_connected",
+ )
def _on_participant_disconnected_wrapper(self, participant: rtc.RemoteParticipant):
- asyncio.create_task(self._async_on_participant_disconnected(participant))
+ self._task_manager.create_task(
+ self._async_on_participant_disconnected(participant),
+ f"{self}::_async_on_participant_disconnected",
+ )
def _on_track_subscribed_wrapper(
self,
@@ -226,7 +245,10 @@ class LiveKitTransportClient:
publication: rtc.RemoteTrackPublication,
participant: rtc.RemoteParticipant,
):
- asyncio.create_task(self._async_on_track_subscribed(track, publication, participant))
+ self._task_manager.create_task(
+ self._async_on_track_subscribed(track, publication, participant),
+ f"{self}::_async_on_track_subscribed",
+ )
def _on_track_unsubscribed_wrapper(
self,
@@ -234,16 +256,24 @@ class LiveKitTransportClient:
publication: rtc.RemoteTrackPublication,
participant: rtc.RemoteParticipant,
):
- asyncio.create_task(self._async_on_track_unsubscribed(track, publication, participant))
+ self._task_manager.create_task(
+ self._async_on_track_unsubscribed(track, publication, participant),
+ f"{self}::_async_on_track_unsubscribed",
+ )
def _on_data_received_wrapper(self, data: rtc.DataPacket):
- asyncio.create_task(self._async_on_data_received(data))
+ self._task_manager.create_task(
+ self._async_on_data_received(data),
+ f"{self}::_async_on_data_received",
+ )
def _on_connected_wrapper(self):
- asyncio.create_task(self._async_on_connected())
+ self._task_manager.create_task(self._async_on_connected(), f"{self}::_async_on_connected")
def _on_disconnected_wrapper(self):
- asyncio.create_task(self._async_on_disconnected())
+ self._task_manager.create_task(
+ self._async_on_disconnected(), f"{self}::_async_on_disconnected"
+ )
# Async methods for event handling
async def _async_on_participant_connected(self, participant: rtc.RemoteParticipant):
@@ -269,7 +299,10 @@ class LiveKitTransportClient:
logger.info(f"Audio track subscribed: {track.sid} from participant {participant.sid}")
self._audio_tracks[participant.sid] = track
audio_stream = rtc.AudioStream(track)
- asyncio.create_task(self._process_audio_stream(audio_stream, participant.sid))
+ self._task_manager.create_task(
+ self._process_audio_stream(audio_stream, participant.sid),
+ f"{self}::_process_audio_stream",
+ )
async def _async_on_track_unsubscribed(
self,
@@ -307,6 +340,9 @@ class LiveKitTransportClient:
frame, participant_id = await self._audio_queue.get()
return frame, participant_id
+ def __str__(self):
+ return f"{self._transport_name}::LiveKitTransportClient"
+
class LiveKitInputTransport(BaseInputTransport):
def __init__(self, client: LiveKitTransportClient, params: LiveKitParams, **kwargs):
@@ -314,31 +350,32 @@ class LiveKitInputTransport(BaseInputTransport):
self._client = client
self._audio_in_task = None
self._vad_analyzer: VADAnalyzer | None = params.vad_analyzer
+ self._resampler = create_default_resampler()
+
+ @property
+ def vad_analyzer(self) -> VADAnalyzer | None:
+ return self._vad_analyzer
async def start(self, frame: StartFrame):
await super().start(frame)
+ await self._client.setup(frame)
await self._client.connect()
if self._params.audio_in_enabled or self._params.vad_enabled:
- self._audio_in_task = asyncio.create_task(self._audio_in_task_handler())
+ self._audio_in_task = self.create_task(self._audio_in_task_handler())
logger.info("LiveKitInputTransport started")
async def stop(self, frame: EndFrame):
await super().stop(frame)
await self._client.disconnect()
if self._audio_in_task:
- self._audio_in_task.cancel()
- await self._audio_in_task
+ await self.cancel_task(self._audio_in_task)
logger.info("LiveKitInputTransport stopped")
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
await self._client.disconnect()
if self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled):
- self._audio_in_task.cancel()
- await self._audio_in_task
-
- def vad_analyzer(self) -> VADAnalyzer | None:
- return self._vad_analyzer
+ await self.cancel_task(self._audio_in_task)
async def push_app_message(self, message: Any, sender: str):
frame = LiveKitTransportMessageUrgentFrame(message=message, participant_id=sender)
@@ -347,38 +384,31 @@ class LiveKitInputTransport(BaseInputTransport):
async def _audio_in_task_handler(self):
logger.info("Audio input task started")
while True:
- try:
- audio_data = await self._client.get_next_audio_frame()
- if audio_data:
- audio_frame_event, participant_id = audio_data
- pipecat_audio_frame = self._convert_livekit_audio_to_pipecat(audio_frame_event)
- input_audio_frame = InputAudioRawFrame(
- audio=pipecat_audio_frame.audio,
- sample_rate=pipecat_audio_frame.sample_rate,
- num_channels=pipecat_audio_frame.num_channels,
- )
- await self.push_audio_frame(input_audio_frame)
- except asyncio.CancelledError:
- logger.info("Audio input task cancelled")
- break
- except Exception as e:
- logger.error(f"Error in audio input task: {e}")
+ audio_data = await self._client.get_next_audio_frame()
+ if audio_data:
+ audio_frame_event, participant_id = audio_data
+ pipecat_audio_frame = await self._convert_livekit_audio_to_pipecat(
+ audio_frame_event
+ )
+ input_audio_frame = InputAudioRawFrame(
+ audio=pipecat_audio_frame.audio,
+ sample_rate=pipecat_audio_frame.sample_rate,
+ num_channels=pipecat_audio_frame.num_channels,
+ )
+ await self.push_audio_frame(input_audio_frame)
- def _convert_livekit_audio_to_pipecat(
+ async def _convert_livekit_audio_to_pipecat(
self, audio_frame_event: rtc.AudioFrameEvent
) -> AudioRawFrame:
audio_frame = audio_frame_event.frame
- audio_data = audio_frame.data
- original_sample_rate = audio_frame.sample_rate
- if original_sample_rate != self._params.audio_in_sample_rate:
- audio_data = resample_audio(
- audio_data, original_sample_rate, self._params.audio_in_sample_rate
- )
+ audio_data = await self._resampler.resample(
+ audio_frame.data.tobytes(), audio_frame.sample_rate, self.sample_rate
+ )
return AudioRawFrame(
audio=audio_data,
- sample_rate=self._params.audio_in_sample_rate,
+ sample_rate=self.sample_rate,
num_channels=audio_frame.num_channels,
)
@@ -390,6 +420,7 @@ class LiveKitOutputTransport(BaseOutputTransport):
async def start(self, frame: StartFrame):
await super().start(frame)
+ await self._client.setup(frame)
await self._client.connect()
logger.info("LiveKitOutputTransport started")
@@ -419,7 +450,7 @@ class LiveKitOutputTransport(BaseOutputTransport):
return rtc.AudioFrame(
data=pipecat_audio,
- sample_rate=self._params.audio_out_sample_rate,
+ sample_rate=self.sample_rate,
num_channels=self._params.audio_out_channels,
samples_per_channel=samples_per_channel,
)
@@ -434,9 +465,8 @@ class LiveKitTransport(BaseTransport):
params: LiveKitParams = LiveKitParams(),
input_name: str | None = None,
output_name: str | None = None,
- loop: asyncio.AbstractEventLoop | None = None,
):
- super().__init__(input_name=input_name, output_name=output_name, loop=loop)
+ super().__init__(input_name=input_name, output_name=output_name)
callbacks = LiveKitCallbacks(
on_connected=self._on_connected,
@@ -451,7 +481,7 @@ class LiveKitTransport(BaseTransport):
self._params = params
self._client = LiveKitTransportClient(
- url, token, room_name, self._params, callbacks, self._loop
+ url, token, room_name, self._params, callbacks, self.name
)
self._input: LiveKitInputTransport | None = None
self._output: LiveKitOutputTransport | None = None
@@ -517,7 +547,7 @@ class LiveKitTransport(BaseTransport):
async def _on_audio_track_subscribed(self, participant_id: str):
await self._call_event_handler("on_audio_track_subscribed", participant_id)
- participant = self._client._room.remote_participants.get(participant_id)
+ participant = self._client.room.remote_participants.get(participant_id)
if participant:
for publication in participant.audio_tracks.values():
self._client._on_track_subscribed_wrapper(
diff --git a/src/pipecat/utils/asyncio.py b/src/pipecat/utils/asyncio.py
new file mode 100644
index 000000000..ab40a8195
--- /dev/null
+++ b/src/pipecat/utils/asyncio.py
@@ -0,0 +1,129 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import asyncio
+from typing import Coroutine, Optional, Set
+
+from loguru import logger
+
+
+class TaskManager:
+ def __init__(self) -> None:
+ self._tasks: Set[asyncio.Task] = set()
+ self._loop: Optional[asyncio.AbstractEventLoop] = None
+
+ def set_event_loop(self, loop: asyncio.AbstractEventLoop):
+ self._loop = loop
+
+ def get_event_loop(self) -> asyncio.AbstractEventLoop:
+ if not self._loop:
+ raise Exception("TaskManager missing event loop, use TaskManager.set_event_loop().")
+ return self._loop
+
+ def create_task(self, coroutine: Coroutine, name: str) -> asyncio.Task:
+ """
+ Creates and schedules a new asyncio Task that runs the given coroutine.
+
+ The task is added to a global set of created tasks.
+
+ Args:
+ loop (asyncio.AbstractEventLoop): The event loop to use for creating the task.
+ coroutine (Coroutine): The coroutine to be executed within the task.
+ name (str): The name to assign to the task for identification.
+
+ Returns:
+ asyncio.Task: The created task object.
+ """
+
+ async def run_coroutine():
+ try:
+ await coroutine
+ except asyncio.CancelledError:
+ logger.trace(f"{name}: task cancelled")
+ # Re-raise the exception to ensure the task is cancelled.
+ raise
+ except Exception as e:
+ logger.exception(f"{name}: unexpected exception: {e}")
+
+ if not self._loop:
+ raise Exception("TaskManager missing event loop, use TaskManager.set_event_loop().")
+
+ task = self._loop.create_task(run_coroutine())
+ task.set_name(name)
+ self._add_task(task)
+ logger.trace(f"{name}: task created")
+ return task
+
+ async def wait_for_task(self, task: asyncio.Task, timeout: Optional[float] = None):
+ """Wait for an asyncio.Task to complete with optional timeout handling.
+
+ This function awaits the specified asyncio.Task and handles scenarios for
+ timeouts, cancellations, and other exceptions. It also ensures that the task
+ is removed from the set of registered tasks upon completion or failure.
+
+ Args:
+ task (asyncio.Task): The asyncio Task to wait for.
+ timeout (Optional[float], optional): The maximum number of seconds
+ to wait for the task to complete. If None, waits indefinitely.
+ Defaults to None.
+ """
+ name = task.get_name()
+ try:
+ if timeout:
+ await asyncio.wait_for(task, timeout=timeout)
+ else:
+ await task
+ except asyncio.TimeoutError:
+ logger.warning(f"{name}: timed out waiting for task to finish")
+ except asyncio.CancelledError:
+ logger.trace(f"{name}: unexpected task cancellation (maybe Ctrl-C?)")
+ except Exception as e:
+ logger.exception(f"{name}: unexpected exception while stopping task: {e}")
+ finally:
+ self._remove_task(task)
+
+ async def cancel_task(self, task: asyncio.Task, timeout: Optional[float] = None):
+ """Cancels the given asyncio Task and awaits its completion with an
+ optional timeout.
+
+ This function removes the task from the set of registered tasks upon
+ completion or failure.
+
+ Args:
+ task (asyncio.Task): The task to be cancelled.
+ timeout (Optional[float]): The optional timeout in seconds to wait for the task to cancel.
+
+ """
+ name = task.get_name()
+ task.cancel()
+ try:
+ if timeout:
+ await asyncio.wait_for(task, timeout=timeout)
+ else:
+ await task
+ except asyncio.TimeoutError:
+ logger.warning(f"{name}: timed out waiting for task to cancel")
+ except asyncio.CancelledError:
+ # Here are sure the task is cancelled properly.
+ pass
+ except Exception as e:
+ logger.exception(f"{name}: unexpected exception while cancelling task: {e}")
+ finally:
+ self._remove_task(task)
+
+ def current_tasks(self) -> Set[asyncio.Task]:
+ """Returns the list of currently created/registered tasks."""
+ return self._tasks
+
+ def _add_task(self, task: asyncio.Task):
+ self._tasks.add(task)
+
+ def _remove_task(self, task: asyncio.Task):
+ name = task.get_name()
+ try:
+ self._tasks.remove(task)
+ except KeyError as e:
+ logger.trace(f"{name}: unable to remove task (already removed?): {e}")
diff --git a/src/pipecat/utils/string.py b/src/pipecat/utils/string.py
index c901586f7..c269c5e57 100644
--- a/src/pipecat/utils/string.py
+++ b/src/pipecat/utils/string.py
@@ -14,7 +14,7 @@ ENDOFSENTENCE_PATTERN_STR = r"""
(? int:
@@ -20,7 +24,8 @@ def obj_id() -> int:
>>> obj_id()
2
"""
- return next(_ID)
+ with _ID_LOCK:
+ return next(_ID)
def obj_count(obj) -> int:
@@ -34,4 +39,5 @@ def obj_count(obj) -> int:
>>> obj_count(new_type())
0
"""
- return next(_COUNTS[obj.__class__.__name__])
+ with _COUNTS_LOCK:
+ return next(_COUNTS[obj.__class__.__name__])
diff --git a/test-requirements.txt b/test-requirements.txt
index f974121b0..36e64060b 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -22,8 +22,8 @@ pydantic~=2.8.2
pyloudnorm~=0.1.1
pyht~=0.1.4
python-dotenv~=1.0.1
-resampy~=0.4.3
silero-vad~=5.1
+soxr~=0.5.0
together~=1.2.7
transformers~=4.44.0
websockets~=13.1
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/test_aggregators.py b/tests/test_aggregators.py
index dcf27ad6e..ab7210e49 100644
--- a/tests/test_aggregators.py
+++ b/tests/test_aggregators.py
@@ -1,122 +1,76 @@
-import asyncio
-import doctest
-import functools
+#
+# Copyright (c) 2024-2025 Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
import unittest
from pipecat.frames.frames import (
- AudioRawFrame,
- EndFrame,
- Frame,
ImageRawFrame,
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
+ OutputAudioRawFrame,
+ OutputImageRawFrame,
TextFrame,
)
-from pipecat.pipeline.parallel_pipeline import ParallelPipeline
-from pipecat.pipeline.pipeline import Pipeline
from pipecat.processors.aggregators.gated import GatedAggregator
from pipecat.processors.aggregators.sentence import SentenceAggregator
-from pipecat.processors.text_transformer import StatelessTextTransformer
+from pipecat.tests.utils import run_test
-class TestDailyFrameAggregators(unittest.IsolatedAsyncioTestCase):
- @unittest.skip("FIXME: This test is failing")
+class TestSentenceAggregator(unittest.IsolatedAsyncioTestCase):
async def test_sentence_aggregator(self):
- sentence = "Hello, world. How are you? I am fine"
- expected_sentences = ["Hello, world.", " How are you?", " I am fine "]
aggregator = SentenceAggregator()
+
+ sentence = "Hello, world. How are you? I am fine!"
+
+ frames_to_send = []
for word in sentence.split(" "):
- async for sentence in aggregator.process_frame(TextFrame(word + " ")):
- self.assertIsInstance(sentence, TextFrame)
- if isinstance(sentence, TextFrame):
- self.assertEqual(sentence.text, expected_sentences.pop(0))
+ frames_to_send.append(TextFrame(text=word + " "))
- async for sentence in aggregator.process_frame(EndFrame()):
- if len(expected_sentences):
- self.assertIsInstance(sentence, TextFrame)
- if isinstance(sentence, TextFrame):
- self.assertEqual(sentence.text, expected_sentences.pop(0))
- else:
- self.assertIsInstance(sentence, EndFrame)
+ expected_returned_frames = [TextFrame, TextFrame, TextFrame]
- self.assertEqual(expected_sentences, [])
+ (received_down, _) = await run_test(
+ aggregator,
+ frames_to_send=frames_to_send,
+ expected_down_frames=expected_returned_frames,
+ )
+ assert received_down[-3].text == "Hello, world. "
+ assert received_down[-2].text == "How are you? "
+ assert received_down[-1].text == "I am fine! "
- @unittest.skip("FIXME: This test is failing")
- async def test_gated_accumulator(self):
+
+class TestGatedAggregator(unittest.IsolatedAsyncioTestCase):
+ async def test_gated_aggregator(self):
gated_aggregator = GatedAggregator(
gate_open_fn=lambda frame: isinstance(frame, ImageRawFrame),
gate_close_fn=lambda frame: isinstance(frame, LLMFullResponseStartFrame),
start_open=False,
)
- frames = [
+ frames_to_send = [
LLMFullResponseStartFrame(),
TextFrame("Hello, "),
TextFrame("world."),
- AudioRawFrame(b"hello"),
- ImageRawFrame(b"image", (0, 0)),
- AudioRawFrame(b"world"),
+ OutputAudioRawFrame(audio=b"hello", sample_rate=16000, num_channels=1),
+ OutputImageRawFrame(image=b"image", size=(0, 0), format="RGB"),
+ OutputAudioRawFrame(audio=b"world", sample_rate=16000, num_channels=1),
LLMFullResponseEndFrame(),
]
- expected_output_frames = [
- ImageRawFrame(b"image", (0, 0)),
- LLMFullResponseStartFrame(),
- TextFrame("Hello, "),
- TextFrame("world."),
- AudioRawFrame(b"hello"),
- AudioRawFrame(b"world"),
- LLMFullResponseEndFrame(),
+ expected_returned_frames = [
+ OutputImageRawFrame,
+ LLMFullResponseStartFrame,
+ TextFrame,
+ TextFrame,
+ OutputAudioRawFrame,
+ OutputAudioRawFrame,
+ LLMFullResponseEndFrame,
]
- for frame in frames:
- async for out_frame in gated_aggregator.process_frame(frame):
- self.assertEqual(out_frame, expected_output_frames.pop(0))
- self.assertEqual(expected_output_frames, [])
- @unittest.skip("FIXME: This test is failing")
- async def test_parallel_pipeline(self):
- async def slow_add(sleep_time: float, name: str, x: str):
- await asyncio.sleep(sleep_time)
- return ":".join([x, name])
-
- pipe1_annotation = StatelessTextTransformer(functools.partial(slow_add, 0.1, "pipe1"))
- pipe2_annotation = StatelessTextTransformer(functools.partial(slow_add, 0.2, "pipe2"))
- sentence_aggregator = SentenceAggregator()
- add_dots = StatelessTextTransformer(lambda x: x + ".")
-
- source = asyncio.Queue()
- sink = asyncio.Queue()
- pipeline = Pipeline(
- [
- ParallelPipeline([[pipe1_annotation], [sentence_aggregator, pipe2_annotation]]),
- add_dots,
- ],
- source,
- sink,
+ (received_down, _) = await run_test(
+ gated_aggregator,
+ frames_to_send=frames_to_send,
+ expected_down_frames=expected_returned_frames,
)
-
- frames = [TextFrame("Hello, "), TextFrame("world."), EndFrame()]
-
- expected_output_frames: list[Frame] = [
- TextFrame(text="Hello, :pipe1."),
- TextFrame(text="world.:pipe1."),
- TextFrame(text="Hello, world.:pipe2."),
- EndFrame(),
- ]
-
- for frame in frames:
- await source.put(frame)
-
- await pipeline.run_pipeline()
-
- while not sink.empty():
- frame = await sink.get()
- self.assertEqual(frame, expected_output_frames.pop(0))
-
-
-def load_tests(loader, tests, ignore):
- """Run doctests on the aggregators module."""
- from pipecat.processors import aggregators
-
- tests.addTests(doctest.DocTestSuite(aggregators))
- return tests
diff --git a/tests/test_daily_transport_service.py b/tests/test_daily_transport_service.py
index 8c2788c9e..aabbd733d 100644
--- a/tests/test_daily_transport_service.py
+++ b/tests/test_daily_transport_service.py
@@ -1,3 +1,9 @@
+#
+# Copyright (c) 2024-2025 Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
import unittest
diff --git a/tests/test_filters.py b/tests/test_filters.py
new file mode 100644
index 000000000..9d3b0f003
--- /dev/null
+++ b/tests/test_filters.py
@@ -0,0 +1,120 @@
+#
+# Copyright (c) 2024-2025 Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import unittest
+
+from pipecat.frames.frames import (
+ EndFrame,
+ Frame,
+ TextFrame,
+ TranscriptionFrame,
+ UserStartedSpeakingFrame,
+ UserStoppedSpeakingFrame,
+)
+from pipecat.processors.filters.frame_filter import FrameFilter
+from pipecat.processors.filters.function_filter import FunctionFilter
+from pipecat.processors.filters.identity_filter import IdentityFilter
+from pipecat.processors.filters.wake_check_filter import WakeCheckFilter
+from pipecat.tests.utils import run_test
+
+
+class TestIdentifyFilter(unittest.IsolatedAsyncioTestCase):
+ async def test_identity(self):
+ filter = IdentityFilter()
+ frames_to_send = [UserStartedSpeakingFrame(), UserStoppedSpeakingFrame()]
+ expected_returned_frames = [UserStartedSpeakingFrame, UserStoppedSpeakingFrame]
+ await run_test(
+ filter,
+ frames_to_send=frames_to_send,
+ expected_down_frames=expected_returned_frames,
+ )
+
+
+class TestFrameFilter(unittest.IsolatedAsyncioTestCase):
+ async def test_text_frame(self):
+ filter = FrameFilter(types=(TextFrame,))
+ frames_to_send = [TextFrame(text="Hello Pipecat!")]
+ expected_returned_frames = [TextFrame]
+ await run_test(
+ filter,
+ frames_to_send=frames_to_send,
+ expected_down_frames=expected_returned_frames,
+ )
+
+ async def test_end_frame(self):
+ filter = FrameFilter(types=(EndFrame,))
+ frames_to_send = [EndFrame()]
+ expected_returned_frames = [EndFrame]
+ await run_test(
+ filter,
+ frames_to_send=frames_to_send,
+ expected_down_frames=expected_returned_frames,
+ send_end_frame=False,
+ )
+
+ async def test_system_frame(self):
+ filter = FrameFilter(types=())
+ frames_to_send = [UserStartedSpeakingFrame()]
+ expected_returned_frames = [UserStartedSpeakingFrame]
+ await run_test(
+ filter,
+ frames_to_send=frames_to_send,
+ expected_down_frames=expected_returned_frames,
+ )
+
+
+class TestFunctionFilter(unittest.IsolatedAsyncioTestCase):
+ async def test_passthrough(self):
+ async def passthrough(frame: Frame):
+ return True
+
+ filter = FunctionFilter(filter=passthrough)
+ frames_to_send = [TextFrame(text="Hello Pipecat!")]
+ expected_returned_frames = [TextFrame]
+ await run_test(
+ filter,
+ frames_to_send=frames_to_send,
+ expected_down_frames=expected_returned_frames,
+ )
+
+ async def test_no_passthrough(self):
+ async def no_passthrough(frame: Frame):
+ return False
+
+ filter = FunctionFilter(filter=no_passthrough)
+ frames_to_send = [TextFrame(text="Hello Pipecat!")]
+ expected_returned_frames = []
+ await run_test(
+ filter,
+ frames_to_send=frames_to_send,
+ expected_down_frames=expected_returned_frames,
+ )
+
+
+class TestWakeCheckFilter(unittest.IsolatedAsyncioTestCase):
+ async def test_no_wake_word(self):
+ filter = WakeCheckFilter(wake_phrases=["Hey, Pipecat"])
+ frames_to_send = [TranscriptionFrame(user_id="test", text="Phrase 1", timestamp="")]
+ expected_returned_frames = []
+ await run_test(
+ filter,
+ frames_to_send=frames_to_send,
+ expected_down_frames=expected_returned_frames,
+ )
+
+ async def test_wake_word(self):
+ filter = WakeCheckFilter(wake_phrases=["Hey, Pipecat"])
+ frames_to_send = [
+ TranscriptionFrame(user_id="test", text="Hey, Pipecat", timestamp=""),
+ TranscriptionFrame(user_id="test", text="Phrase 1", timestamp=""),
+ ]
+ expected_returned_frames = [TranscriptionFrame, TranscriptionFrame]
+ (received_down, _) = await run_test(
+ filter,
+ frames_to_send=frames_to_send,
+ expected_down_frames=expected_returned_frames,
+ )
+ assert received_down[-1].text == "Phrase 1"
diff --git a/tests/test_langchain.py b/tests/test_langchain.py
index 22dc9d38d..c94bc1c01 100644
--- a/tests/test_langchain.py
+++ b/tests/test_langchain.py
@@ -32,8 +32,7 @@ from pipecat.processors.frameworks.langchain import LangchainProcessor
class TestLangchain(unittest.IsolatedAsyncioTestCase):
class MockProcessor(FrameProcessor):
def __init__(self, name):
- super().__init__()
- self.name = name
+ super().__init__(name=name)
self.token: list[str] = []
# Start collecting tokens when we see the start frame
self.start_collecting = False
@@ -93,7 +92,3 @@ class TestLangchain(unittest.IsolatedAsyncioTestCase):
# This next one would fail with:
# AssertionError: ' H e l l o d e a r h u m a n' != 'Hello dear human'
# self.assertEqual(tma_out.messages[-1]["content"], self.expected_response)
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/tests/test_openai_tts.py b/tests/test_openai_tts.py
deleted file mode 100644
index 1dc3929a6..000000000
--- a/tests/test_openai_tts.py
+++ /dev/null
@@ -1,37 +0,0 @@
-import asyncio
-import unittest
-
-import openai
-import pyaudio
-from dotenv import load_dotenv
-
-from pipecat.frames.frames import AudioRawFrame, ErrorFrame
-from pipecat.services.openai import OpenAITTSService
-
-load_dotenv()
-
-
-class TestWhisperOpenAIService(unittest.IsolatedAsyncioTestCase):
- @unittest.skip("FIXME: This test is failing")
- async def test_whisper_tts(self):
- pa = pyaudio.PyAudio()
- stream = pa.open(format=pyaudio.paInt16, channels=1, rate=24_000, output=True)
-
- tts = OpenAITTSService(voice="nova")
-
- async for frame in tts.run_tts("Hello, there. Nice to meet you, seems to work well"):
- self.assertIsInstance(frame, AudioRawFrame)
- stream.write(frame.audio)
-
- await asyncio.sleep(0.5)
- stream.stop_stream()
- pa.terminate()
-
- tts = OpenAITTSService(voice="invalid_voice")
- with self.assertRaises(openai.BadRequestError):
- async for frame in tts.run_tts("wont work"):
- self.assertIsInstance(frame, ErrorFrame)
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py
index 7d703d5ed..caac89c11 100644
--- a/tests/test_pipeline.py
+++ b/tests/test_pipeline.py
@@ -1,116 +1,126 @@
+#
+# Copyright (c) 2024-2025 Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
import asyncio
import unittest
-from unittest.mock import Mock
-from pipecat.frames.frames import EndFrame, TextFrame
+from pipecat.frames.frames import EndFrame, HeartbeatFrame, StartFrame, TextFrame
+from pipecat.pipeline.parallel_pipeline import ParallelPipeline
from pipecat.pipeline.pipeline import Pipeline
-from pipecat.processors.aggregators.sentence import SentenceAggregator
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.filters.identity_filter import IdentityFilter
from pipecat.processors.frame_processor import FrameProcessor
-from pipecat.processors.text_transformer import StatelessTextTransformer
+from pipecat.tests.utils import HeartbeatsObserver, run_test
-class TestDailyPipeline(unittest.IsolatedAsyncioTestCase):
- @unittest.skip("FIXME: This test is failing")
- async def test_pipeline_simple(self):
- aggregator = SentenceAggregator()
+class TestPipeline(unittest.IsolatedAsyncioTestCase):
+ async def test_pipeline_single(self):
+ pipeline = Pipeline([IdentityFilter()])
- outgoing_queue = asyncio.Queue()
- incoming_queue = asyncio.Queue()
- pipeline = Pipeline([aggregator], incoming_queue, outgoing_queue)
-
- await incoming_queue.put(TextFrame("Hello, "))
- await incoming_queue.put(TextFrame("world."))
- await incoming_queue.put(EndFrame())
-
- await pipeline.run_pipeline()
-
- self.assertEqual(await outgoing_queue.get(), TextFrame("Hello, world."))
- self.assertIsInstance(await outgoing_queue.get(), EndFrame)
-
- @unittest.skip("FIXME: This test is failing")
- async def test_pipeline_multiple_stages(self):
- sentence_aggregator = SentenceAggregator()
- to_upper = StatelessTextTransformer(lambda x: x.upper())
- add_space = StatelessTextTransformer(lambda x: x + " ")
-
- outgoing_queue = asyncio.Queue()
- incoming_queue = asyncio.Queue()
- pipeline = Pipeline(
- [add_space, sentence_aggregator, to_upper], incoming_queue, outgoing_queue
+ frames_to_send = [TextFrame(text="Hello from Pipecat!")]
+ expected_returned_frames = [TextFrame]
+ await run_test(
+ pipeline,
+ frames_to_send=frames_to_send,
+ expected_down_frames=expected_returned_frames,
)
- sentence = "Hello, world. It's me, a pipeline."
- for c in sentence:
- await incoming_queue.put(TextFrame(c))
- await incoming_queue.put(EndFrame())
+ async def test_pipeline_multiple(self):
+ identity1 = IdentityFilter()
+ identity2 = IdentityFilter()
+ identity3 = IdentityFilter()
- await pipeline.run_pipeline()
+ pipeline = Pipeline([identity1, identity2, identity3])
- self.assertEqual(await outgoing_queue.get(), TextFrame("H E L L O , W O R L D ."))
- self.assertEqual(
- await outgoing_queue.get(),
- TextFrame(" I T ' S M E , A P I P E L I N E ."),
- )
- # leftover little bit because of the spacing
- self.assertEqual(
- await outgoing_queue.get(),
- TextFrame(" "),
- )
- self.assertIsInstance(await outgoing_queue.get(), EndFrame)
-
-
-class TestLogFrame(unittest.TestCase):
- class MockProcessor(FrameProcessor):
- def __init__(self, name):
- self.name = name
-
- def __str__(self):
- return self.name
-
- def setUp(self):
- self.processor1 = self.MockProcessor("processor1")
- self.processor2 = self.MockProcessor("processor2")
- self.pipeline = Pipeline(processors=[self.processor1, self.processor2])
- self.pipeline._name = "MyClass"
- self.pipeline._logger = Mock()
-
- @unittest.skip("FIXME: This test is failing")
- def test_log_frame_from_source(self):
- frame = Mock(__class__=Mock(__name__="MyFrame"))
- self.pipeline._log_frame(frame, depth=1)
- self.pipeline._logger.debug.assert_called_once_with(
- "MyClass source -> MyFrame -> processor1"
+ frames_to_send = [TextFrame(text="Hello from Pipecat!")]
+ expected_returned_frames = [TextFrame]
+ await run_test(
+ pipeline,
+ frames_to_send=frames_to_send,
+ expected_down_frames=expected_returned_frames,
)
- @unittest.skip("FIXME: This test is failing")
- def test_log_frame_to_sink(self):
- frame = Mock(__class__=Mock(__name__="MyFrame"))
- self.pipeline._log_frame(frame, depth=3)
- self.pipeline._logger.debug.assert_called_once_with(
- "MyClass processor2 -> MyFrame -> sink"
+ async def test_pipeline_start_metadata(self):
+ pipeline = Pipeline([IdentityFilter()])
+
+ frames_to_send = []
+ expected_returned_frames = [StartFrame]
+ (received_down, _) = await run_test(
+ pipeline,
+ frames_to_send=frames_to_send,
+ expected_down_frames=expected_returned_frames,
+ ignore_start=False,
+ start_metadata={"foo": "bar"},
+ )
+ assert "foo" in received_down[-1].metadata
+
+
+class TestParallelPipeline(unittest.IsolatedAsyncioTestCase):
+ async def test_parallel_single(self):
+ pipeline = ParallelPipeline([IdentityFilter()])
+
+ frames_to_send = [TextFrame(text="Hello from Pipecat!")]
+ expected_returned_frames = [TextFrame]
+ await run_test(
+ pipeline,
+ frames_to_send=frames_to_send,
+ expected_down_frames=expected_returned_frames,
)
- @unittest.skip("FIXME: This test is failing")
- def test_log_frame_repeated_log(self):
- frame = Mock(__class__=Mock(__name__="MyFrame"))
- self.pipeline._log_frame(frame, depth=2)
- self.pipeline._logger.debug.assert_called_once_with(
- "MyClass processor1 -> MyFrame -> processor2"
- )
- self.pipeline._log_frame(frame, depth=2)
- self.pipeline._logger.debug.assert_called_with("MyClass ... repeated")
+ async def test_parallel_multiple(self):
+ """Should only passthrough one instance of TextFrame."""
+ pipeline = ParallelPipeline([IdentityFilter()], [IdentityFilter()])
- @unittest.skip("FIXME: This test is failing")
- def test_log_frame_reset_repeated_log(self):
- frame1 = Mock(__class__=Mock(__name__="MyFrame1"))
- frame2 = Mock(__class__=Mock(__name__="MyFrame2"))
- self.pipeline._log_frame(frame1, depth=2)
- self.pipeline._logger.debug.assert_called_once_with(
- "MyClass processor1 -> MyFrame1 -> processor2"
+ frames_to_send = [TextFrame(text="Hello from Pipecat!")]
+ expected_returned_frames = [TextFrame]
+ await run_test(
+ pipeline,
+ frames_to_send=frames_to_send,
+ expected_down_frames=expected_returned_frames,
)
- self.pipeline._log_frame(frame1, depth=2)
- self.pipeline._logger.debug.assert_called_with("MyClass ... repeated")
- self.pipeline._log_frame(frame2, depth=2)
- self.pipeline._logger.debug.assert_called_with(
- "MyClass processor1 -> MyFrame2 -> processor2"
+
+
+class TestPipelineTask(unittest.IsolatedAsyncioTestCase):
+ async def test_task_single(self):
+ pipeline = Pipeline([IdentityFilter()])
+ task = PipelineTask(pipeline)
+ task.set_event_loop(asyncio.get_event_loop())
+
+ await task.queue_frame(TextFrame(text="Hello!"))
+ await task.queue_frames([TextFrame(text="Bye!"), EndFrame()])
+ await task.run()
+ assert task.has_finished()
+
+ async def test_task_heartbeats(self):
+ heartbeats_counter = 0
+
+ async def heartbeat_received(processor: FrameProcessor, heartbeat: HeartbeatFrame):
+ nonlocal heartbeats_counter
+ heartbeats_counter += 1
+
+ identity = IdentityFilter()
+ pipeline = Pipeline([identity])
+ heartbeats_observer = HeartbeatsObserver(
+ target=identity, heartbeat_callback=heartbeat_received
)
+ task = PipelineTask(
+ pipeline,
+ params=PipelineParams(
+ enable_heartbeats=True,
+ heartbeats_period_secs=0.2,
+ observers=[heartbeats_observer],
+ ),
+ )
+ task.set_event_loop(asyncio.get_event_loop())
+
+ expected_heartbeats = 1.0 / 0.2
+
+ await task.queue_frame(TextFrame(text="Hello!"))
+ try:
+ await asyncio.wait_for(task.run(), timeout=1.0)
+ except asyncio.TimeoutError:
+ pass
+ assert heartbeats_counter == expected_heartbeats
diff --git a/tests/test_protobuf_serializer.py b/tests/test_protobuf_serializer.py
index 7f9841622..7d45800a8 100644
--- a/tests/test_protobuf_serializer.py
+++ b/tests/test_protobuf_serializer.py
@@ -1,6 +1,16 @@
+#
+# Copyright (c) 2024-2025 Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
import unittest
-from pipecat.frames.frames import AudioRawFrame, TextFrame, TranscriptionFrame
+from pipecat.frames.frames import (
+ OutputAudioRawFrame,
+ TextFrame,
+ TranscriptionFrame,
+)
from pipecat.serializers.protobuf import ProtobufFrameSerializer
@@ -8,22 +18,21 @@ class TestProtobufFrameSerializer(unittest.IsolatedAsyncioTestCase):
def setUp(self):
self.serializer = ProtobufFrameSerializer()
- @unittest.skip("FIXME: This test is failing")
async def test_roundtrip(self):
text_frame = TextFrame(text="hello world")
- frame = self.serializer.deserialize(self.serializer.serialize(text_frame))
- self.assertEqual(frame, TextFrame(text="hello world"))
+ frame = await self.serializer.deserialize(await self.serializer.serialize(text_frame))
+ self.assertEqual(text_frame, frame)
transcription_frame = TranscriptionFrame(
- text="Hello there!", participantId="123", timestamp="2021-01-01"
+ text="Hello there!", user_id="123", timestamp="2021-01-01"
+ )
+ frame = await self.serializer.deserialize(
+ await self.serializer.serialize(transcription_frame)
)
- frame = self.serializer.deserialize(self.serializer.serialize(transcription_frame))
self.assertEqual(frame, transcription_frame)
- audio_frame = AudioRawFrame(data=b"1234567890")
- frame = self.serializer.deserialize(self.serializer.serialize(audio_frame))
- self.assertEqual(frame, audio_frame)
-
-
-if __name__ == "__main__":
- unittest.main()
+ audio_frame = OutputAudioRawFrame(audio=b"1234567890", sample_rate=16000, num_channels=1)
+ frame = await self.serializer.deserialize(await self.serializer.serialize(audio_frame))
+ self.assertEqual(frame.audio, audio_frame.audio)
+ self.assertEqual(frame.sample_rate, audio_frame.sample_rate)
+ self.assertEqual(frame.num_channels, audio_frame.num_channels)
diff --git a/tests/test_ai_services.py b/tests/test_utils_string.py
similarity index 63%
rename from tests/test_ai_services.py
rename to tests/test_utils_string.py
index 13aa20467..aa135a639 100644
--- a/tests/test_ai_services.py
+++ b/tests/test_utils_string.py
@@ -1,28 +1,15 @@
+#
+# Copyright (c) 2024-2025 Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
import unittest
-from typing import AsyncGenerator
-from pipecat.frames.frames import EndFrame, Frame, TextFrame
-from pipecat.services.ai_services import AIService, match_endofsentence
+from pipecat.utils.string import match_endofsentence
-class SimpleAIService(AIService):
- async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
- yield frame
-
-
-class TestBaseAIService(unittest.IsolatedAsyncioTestCase):
- async def test_simple_processing(self):
- service = SimpleAIService()
-
- input_frames = [TextFrame("hello"), EndFrame()]
-
- output_frames = []
- for input_frame in input_frames:
- async for output_frame in service.process_frame(input_frame):
- output_frames.append(output_frame)
-
- self.assertEqual(input_frames, output_frames)
-
+class TestUtilsString(unittest.IsolatedAsyncioTestCase):
async def test_endofsentence(self):
assert match_endofsentence("This is a sentence.")
assert match_endofsentence("This is a sentence! ")
@@ -52,6 +39,13 @@ class TestBaseAIService(unittest.IsolatedAsyncioTestCase):
assert match_endofsentence(i)
assert not match_endofsentence("你好，")
-
-if __name__ == "__main__":
- unittest.main()
+ async def test_endofsentence_hi(self):
+ hindi_sentences = [
+ "हैलो।",
+ "हैलो！",
+ "आप खाये हैं？",
+ "सुरक्षा पहले।",
+ ]
+ for i in hindi_sentences:
+ assert match_endofsentence(i)
+ assert not match_endofsentence("हैलो，")
diff --git a/tests/test_websocket_transport.py b/tests/test_websocket_transport.py
index b24caa5b9..c918acd8a 100644
--- a/tests/test_websocket_transport.py
+++ b/tests/test_websocket_transport.py
@@ -1,3 +1,9 @@
+#
+# Copyright (c) 2024-2025 Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
# import asyncio
# import unittest
# from unittest.mock import AsyncMock, patch, Mock
-[](https://pypi.org/project/pipecat-ai) [](https://docs.pipecat.ai) [](https://discord.gg/pipecat) 
-