Compare commits
33 Commits
mb/runner-
...
aleix/exam
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6e518cc704 | ||
|
|
96b1000e52 | ||
|
|
0184a8c231 | ||
|
|
c22866ed58 | ||
|
|
0e533d21be | ||
|
|
6f6f4c3dea | ||
|
|
54ff10ae86 | ||
|
|
77057eb829 | ||
|
|
2b1a7b840d | ||
|
|
e07db88bc0 | ||
|
|
c2282b0e73 | ||
|
|
593bf09d8d | ||
|
|
534ed77ebf | ||
|
|
193299988d | ||
|
|
d589bcb345 | ||
|
|
011ebc2801 | ||
|
|
3a72e94d0c | ||
|
|
d6d39fc873 | ||
|
|
258e83c904 | ||
|
|
061f2086b2 | ||
|
|
a1f3f51168 | ||
|
|
2177a2b805 | ||
|
|
68164415ce | ||
|
|
7646599b66 | ||
|
|
e467eaf130 | ||
|
|
9d6d53629e | ||
|
|
89596cfec4 | ||
|
|
5e338ecaf1 | ||
|
|
62319021f8 | ||
|
|
cccd82a617 | ||
|
|
3a7ea25077 | ||
|
|
694922f627 | ||
|
|
e8faf28e6a |
@@ -1,6 +1,6 @@
|
||||
repos:
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.9.7
|
||||
rev: v0.12.1
|
||||
hooks:
|
||||
- id: ruff
|
||||
language_version: python3
|
||||
|
||||
24
CHANGELOG.md
24
CHANGELOG.md
@@ -7,10 +7,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Changed
|
||||
|
||||
- Updated `daily-python` to 0.19.6.
|
||||
|
||||
- Changed `TavusVideoService` to send audio or video frames only after the
|
||||
transport is ready, preventing warning messages at startup.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed an issue in `LiveKitTransport` where empty `AudioRawFrame`s were pushed
|
||||
down the pipeline. This resulted in warnings by the STT processor.
|
||||
- Fixed `PiperTTSService` to send text as a JSON object in the request body,
|
||||
resolving compatibility with Piper's HTTP API.
|
||||
|
||||
- Fixed an issue with the `TavusVideoService` where an error was thrown due to
|
||||
missing transcription callbacks.
|
||||
|
||||
- Fixed an issue in `SpeechmaticsSTTService` where the `user_id` was set to
|
||||
`None` when diarization is not enabled.
|
||||
|
||||
### Performance
|
||||
|
||||
- Fixed an issue in `TaskObserver` (a proxy to all observers) that was degrading
|
||||
global performance.
|
||||
|
||||
### Other
|
||||
|
||||
- Allow Daily transport to quickstart bot example.
|
||||
|
||||
## [0.0.77] - 2025-07-31
|
||||
|
||||
|
||||
@@ -49,7 +49,7 @@ async def run_bot(transport: BaseTransport):
|
||||
This example will use diarization within our STT service and output the words spoken by
|
||||
each individual speaker and wrap them with XML tags.
|
||||
|
||||
If you do not wish to use diarization, then set the `enable_speaker_diarization` parameter
|
||||
If you do not wish to use diarization, then set the `enable_diarization` parameter
|
||||
to `False` or omit it altogether. The `text_format` will only be used if diarization is enabled.
|
||||
|
||||
By default, this example will use our ENHANCED operating point, which is optimized for
|
||||
|
||||
@@ -4,6 +4,11 @@
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
# /// script
|
||||
# dependencies = [
|
||||
# "pipecat-ai[daily,webrtc,websocket,runner,silero,deepgram,google,cartesia,heygen]>=0.0.77",
|
||||
# ]
|
||||
# ///
|
||||
|
||||
import os
|
||||
|
||||
|
||||
@@ -32,12 +32,13 @@ from pipecat.pipeline.runner import PipelineRunner
|
||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||
from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor
|
||||
from pipecat.runner.types import RunnerArguments
|
||||
from pipecat.runner.types import DailyRunnerArguments, RunnerArguments, SmallWebRTCRunnerArguments
|
||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||
from pipecat.services.openai.llm import OpenAILLMService
|
||||
from pipecat.transports.base_transport import BaseTransport, TransportParams
|
||||
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
||||
|
||||
load_dotenv(override=True)
|
||||
|
||||
@@ -108,16 +109,33 @@ async def run_bot(transport: BaseTransport):
|
||||
async def bot(runner_args: RunnerArguments):
|
||||
"""Main bot entry point for the bot starter."""
|
||||
|
||||
transport = SmallWebRTCTransport(
|
||||
params=TransportParams(
|
||||
audio_in_enabled=True,
|
||||
audio_out_enabled=True,
|
||||
vad_analyzer=SileroVADAnalyzer(),
|
||||
),
|
||||
webrtc_connection=runner_args.webrtc_connection,
|
||||
)
|
||||
transport = None
|
||||
|
||||
await run_bot(transport)
|
||||
if isinstance(runner_args, SmallWebRTCRunnerArguments):
|
||||
transport = SmallWebRTCTransport(
|
||||
params=TransportParams(
|
||||
audio_in_enabled=True,
|
||||
audio_out_enabled=True,
|
||||
vad_analyzer=SileroVADAnalyzer(),
|
||||
),
|
||||
webrtc_connection=runner_args.webrtc_connection,
|
||||
)
|
||||
elif isinstance(runner_args, DailyRunnerArguments):
|
||||
transport = DailyTransport(
|
||||
runner_args.room_url,
|
||||
runner_args.token,
|
||||
"Pipecat Quickstart",
|
||||
params=DailyParams(
|
||||
audio_in_enabled=True,
|
||||
audio_out_enabled=True,
|
||||
vad_analyzer=SileroVADAnalyzer(),
|
||||
),
|
||||
)
|
||||
else:
|
||||
logger.warning("Unsupported transport")
|
||||
|
||||
if transport:
|
||||
await run_bot(transport)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -52,7 +52,7 @@ azure = [ "azure-cognitiveservices-speech~=1.42.0"]
|
||||
cartesia = [ "cartesia~=2.0.3", "websockets>=13.1,<15.0" ]
|
||||
cerebras = []
|
||||
deepseek = []
|
||||
daily = [ "daily-python~=0.19.5" ]
|
||||
daily = [ "daily-python~=0.19.6" ]
|
||||
deepgram = [ "deepgram-sdk~=4.7.0" ]
|
||||
elevenlabs = [ "websockets>=13.1,<15.0" ]
|
||||
fal = [ "fal-client~=0.5.9" ]
|
||||
|
||||
@@ -153,22 +153,23 @@ class TaskObserver(BaseObserver):
|
||||
|
||||
async def _proxy_task_handler(self, queue: asyncio.Queue, observer: BaseObserver):
|
||||
"""Handle frame processing for a single observer."""
|
||||
warning_reported = False
|
||||
on_push_frame_deprecated = False
|
||||
signature = inspect.signature(observer.on_push_frame)
|
||||
if len(signature.parameters) > 1:
|
||||
import warnings
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("always")
|
||||
warnings.warn(
|
||||
"Observer `on_push_frame(source, destination, frame, direction, timestamp)` is deprecated, us `on_push_frame(data: FramePushed)` instead.",
|
||||
DeprecationWarning,
|
||||
)
|
||||
|
||||
on_push_frame_deprecated = True
|
||||
|
||||
while True:
|
||||
data = await queue.get()
|
||||
|
||||
signature = inspect.signature(observer.on_push_frame)
|
||||
if len(signature.parameters) > 1:
|
||||
if not warning_reported:
|
||||
import warnings
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("always")
|
||||
warnings.warn(
|
||||
"Observer `on_push_frame(source, destination, frame, direction, timestamp)` is deprecated, us `on_push_frame(data: FramePushed)` instead.",
|
||||
DeprecationWarning,
|
||||
)
|
||||
warning_reported = True
|
||||
if on_push_frame_deprecated:
|
||||
await observer.on_push_frame(
|
||||
data.src, data.dst, data.frame, data.direction, data.timestamp
|
||||
)
|
||||
|
||||
@@ -983,17 +983,14 @@ class GeminiMultimodalLiveLLMService(LLMService):
|
||||
with audio and video inputs, preventing temporal misalignment that can occur
|
||||
when different modalities are processed through separate API pathways.
|
||||
|
||||
After sending the text, we signal turn completion to trigger a model response
|
||||
for text-only interactions.
|
||||
For realtimeInput, turn completion is automatically inferred by the API based
|
||||
on user activity, so no explicit turnComplete signal is needed.
|
||||
|
||||
Args:
|
||||
text: The text to send as user input.
|
||||
"""
|
||||
evt = events.TextInputMessage.from_text(text)
|
||||
await self.send_client_event(evt)
|
||||
# After sending text, we need to signal that the turn is complete.
|
||||
evt = events.ClientContentMessage.model_validate({"clientContent": {"turnComplete": True}})
|
||||
await self.send_client_event(evt)
|
||||
|
||||
async def _send_user_video(self, frame):
|
||||
"""Send user video frame to Gemini Live API."""
|
||||
|
||||
@@ -25,7 +25,6 @@ from loguru import logger
|
||||
|
||||
from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
|
||||
from pipecat.adapters.schemas.direct_function import DirectFunction, DirectFunctionWrapper
|
||||
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
||||
from pipecat.adapters.services.open_ai_adapter import OpenAILLMAdapter
|
||||
from pipecat.frames.frames import (
|
||||
CancelFrame,
|
||||
@@ -108,6 +107,7 @@ class FunctionCallRegistryItem:
|
||||
function_name: Optional[str]
|
||||
handler: FunctionCallHandler | "DirectFunctionWrapper"
|
||||
cancel_on_interruption: bool
|
||||
handler_deprecated: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -282,12 +282,25 @@ class LLMService(AIService):
|
||||
cancel_on_interruption: Whether to cancel this function call when an
|
||||
interruption occurs. Defaults to True.
|
||||
"""
|
||||
signature = inspect.signature(handler)
|
||||
handler_deprecated = len(signature.parameters) > 1
|
||||
if handler_deprecated:
|
||||
import warnings
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("always")
|
||||
warnings.warn(
|
||||
"Function calls with parameters `(function_name, tool_call_id, arguments, llm, context, result_callback)` are deprecated, use a single `FunctionCallParams` parameter instead.",
|
||||
DeprecationWarning,
|
||||
)
|
||||
|
||||
# Registering a function with the function_name set to None will run
|
||||
# that handler for all functions
|
||||
self._functions[function_name] = FunctionCallRegistryItem(
|
||||
function_name=function_name,
|
||||
handler=handler,
|
||||
cancel_on_interruption=cancel_on_interruption,
|
||||
handler_deprecated=handler_deprecated,
|
||||
)
|
||||
|
||||
# Start callbacks are now deprecated.
|
||||
@@ -325,6 +338,7 @@ class LLMService(AIService):
|
||||
function_name=wrapper.name,
|
||||
handler=wrapper,
|
||||
cancel_on_interruption=cancel_on_interruption,
|
||||
handler_deprecated=False,
|
||||
)
|
||||
|
||||
def unregister_function(self, function_name: Optional[str]):
|
||||
@@ -552,17 +566,7 @@ class LLMService(AIService):
|
||||
)
|
||||
else:
|
||||
# Handler is a FunctionCallHandler
|
||||
signature = inspect.signature(item.handler)
|
||||
if len(signature.parameters) > 1:
|
||||
import warnings
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("always")
|
||||
warnings.warn(
|
||||
"Function calls with parameters `(function_name, tool_call_id, arguments, llm, context, result_callback)` are deprecated, use a single `FunctionCallParams` parameter instead.",
|
||||
DeprecationWarning,
|
||||
)
|
||||
|
||||
if item.handler_deprecated:
|
||||
await item.handler(
|
||||
runner_item.function_name,
|
||||
runner_item.tool_call_id,
|
||||
|
||||
@@ -84,7 +84,9 @@ class PiperTTSService(TTSService):
|
||||
try:
|
||||
await self.start_ttfb_metrics()
|
||||
|
||||
async with self._session.post(self._base_url, json=text, headers=headers) as response:
|
||||
async with self._session.post(
|
||||
self._base_url, json={"text": text}, headers=headers
|
||||
) as response:
|
||||
if response.status != 200:
|
||||
error = await response.text()
|
||||
logger.error(
|
||||
|
||||
@@ -191,7 +191,7 @@ class SpeakerFragments:
|
||||
passive_format = active_format
|
||||
return {
|
||||
"text": self._format_text(active_format if self.is_active else passive_format),
|
||||
"user_id": self.speaker_id,
|
||||
"user_id": self.speaker_id or "",
|
||||
"timestamp": self.timestamp,
|
||||
"language": self.language,
|
||||
"result": [frag.result for frag in self.fragments],
|
||||
|
||||
@@ -24,6 +24,7 @@ from pipecat.frames.frames import (
|
||||
Frame,
|
||||
OutputAudioRawFrame,
|
||||
OutputImageRawFrame,
|
||||
OutputTransportReadyFrame,
|
||||
StartFrame,
|
||||
StartInterruptionFrame,
|
||||
TTSAudioRawFrame,
|
||||
@@ -81,6 +82,7 @@ class TavusVideoService(AIService):
|
||||
self._send_task: Optional[asyncio.Task] = None
|
||||
# This is the custom track destination expected by Tavus
|
||||
self._transport_destination: Optional[str] = "stream"
|
||||
self._transport_ready = False
|
||||
|
||||
async def setup(self, setup: FrameProcessorSetup):
|
||||
"""Set up the Tavus video service.
|
||||
@@ -145,7 +147,8 @@ class TavusVideoService(AIService):
|
||||
format=video_frame.color_format,
|
||||
)
|
||||
frame.transport_source = video_source
|
||||
await self.push_frame(frame)
|
||||
if self._transport_ready:
|
||||
await self.push_frame(frame)
|
||||
|
||||
async def _on_participant_audio_data(
|
||||
self, participant_id: str, audio: AudioData, audio_source: str
|
||||
@@ -157,7 +160,8 @@ class TavusVideoService(AIService):
|
||||
num_channels=audio.num_channels,
|
||||
)
|
||||
frame.transport_source = audio_source
|
||||
await self.push_frame(frame)
|
||||
if self._transport_ready:
|
||||
await self.push_frame(frame)
|
||||
|
||||
def can_generate_metrics(self) -> bool:
|
||||
"""Check if this service can generate processing metrics.
|
||||
@@ -221,6 +225,9 @@ class TavusVideoService(AIService):
|
||||
await self.push_frame(frame, direction)
|
||||
elif isinstance(frame, TTSAudioRawFrame):
|
||||
await self._handle_audio_frame(frame)
|
||||
elif isinstance(frame, OutputTransportReadyFrame):
|
||||
self._transport_ready = True
|
||||
await self.push_frame(frame, direction)
|
||||
else:
|
||||
await self.push_frame(frame, direction)
|
||||
|
||||
|
||||
@@ -245,6 +245,10 @@ class TavusTransportClient:
|
||||
on_recording_started=partial(self._on_handle_callback, "on_recording_started"),
|
||||
on_recording_stopped=partial(self._on_handle_callback, "on_recording_stopped"),
|
||||
on_recording_error=partial(self._on_handle_callback, "on_recording_error"),
|
||||
on_transcription_stopped=partial(
|
||||
self._on_handle_callback, "on_transcription_stopped"
|
||||
),
|
||||
on_transcription_error=partial(self._on_handle_callback, "on_transcription_error"),
|
||||
)
|
||||
self._client = DailyTransportClient(
|
||||
room_url, None, "Pipecat", self._params, daily_callbacks, self._bot_name
|
||||
|
||||
12
uv.lock
generated
12
uv.lock
generated
@@ -1185,13 +1185,13 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "daily-python"
|
||||
version = "0.19.5"
|
||||
version = "0.19.6"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/2a/ca/683c8a729b43a6e0ac4296973908be1c9cb0956bca69ecd6e5e4c4d56015/daily_python-0.19.5-cp37-abi3-macosx_10_15_x86_64.whl", hash = "sha256:a1c3e70a4dd87d0a829ebbb657c87cd20737246ffe1bc8351010cef8cdb34a52", size = 13686352, upload-time = "2025-07-30T20:19:03.658Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/90/6e/a0e735021a27e81d2f3d26a2664886e06608cda50bd71ddd3d111be39c72/daily_python-0.19.5-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:98faf21a04bd29086245319e3a4df0ba164f0d0224fc4f8278365e70d927ed45", size = 12018783, upload-time = "2025-07-30T20:19:05.921Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8c/60/454e0f7efe7086411bf8a679caf866f036d6b51a68154f5d1cdd22933bfd/daily_python-0.19.5-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9021e012ec3a39faea1afd4af2c1744e2c67e3a5d397358529b375a7c917f756", size = 14055444, upload-time = "2025-07-30T20:19:08.043Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/49/1e/cc4e3d04aef2a84a48ff9c0a8c5e58d0fd617b3ebf0cd9c8f0ab73cd6346/daily_python-0.19.5-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b64d67f513097004d96796562a46020ff982c7633985be124faf59a4157344b0", size = 14568186, upload-time = "2025-07-30T20:19:10.196Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4a/73/74a313e6557a6f4da153b38f3feb5b9e4012268322976c79495ce2f228ec/daily_python-0.19.6-cp37-abi3-macosx_10_15_x86_64.whl", hash = "sha256:d35d181f9ba824b62ca531449b237dc9aa771e69c00205a6efa359c371208ee6", size = 13687145, upload-time = "2025-08-02T01:56:51.197Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bb/ad/fe2f81fc516851200ffec75f8ce44f631b85c9bc5bbdf9f6637f0823e99a/daily_python-0.19.6-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:826342ee863086bb7192230615e26920700706ea79824f7028305f084e18d2e9", size = 12045258, upload-time = "2025-08-02T01:56:53.38Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/22/0e/52aa355208d015134c88055ed53eed83452204590ac20ee2dcd1d6b4fa95/daily_python-0.19.6-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6ff87c59e677c04c33e46963517e566a77fba425e9d52048418db4e037c6a1d6", size = 14066192, upload-time = "2025-08-02T01:56:55.669Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/32/a0/cab8d478c5ae576bc607431a4d8d9e5a0aae831c096cf833ba2aa1535501/daily_python-0.19.6-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:d15bf8657105a92ddbc88da4dd5bded88075cdf2bc33552406b93f586fdaeced", size = 14575791, upload-time = "2025-08-02T01:56:57.983Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4200,7 +4200,7 @@ requires-dist = [
|
||||
{ name = "azure-cognitiveservices-speech", marker = "extra == 'azure'", specifier = "~=1.42.0" },
|
||||
{ name = "cartesia", marker = "extra == 'cartesia'", specifier = "~=2.0.3" },
|
||||
{ name = "coremltools", marker = "extra == 'local-smart-turn'", specifier = ">=8.0" },
|
||||
{ name = "daily-python", marker = "extra == 'daily'", specifier = "~=0.19.5" },
|
||||
{ name = "daily-python", marker = "extra == 'daily'", specifier = "~=0.19.6" },
|
||||
{ name = "deepgram-sdk", marker = "extra == 'deepgram'", specifier = "~=4.7.0" },
|
||||
{ name = "docstring-parser", specifier = "~=0.16" },
|
||||
{ name = "einops", marker = "extra == 'moondream'", specifier = "~=0.8.0" },
|
||||
|
||||
Reference in New Issue
Block a user