Compare commits
21 Commits
khk/audio-
...
aleix/exam
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6e518cc704 | ||
|
|
96b1000e52 | ||
|
|
0184a8c231 | ||
|
|
c22866ed58 | ||
|
|
0e533d21be | ||
|
|
6f6f4c3dea | ||
|
|
54ff10ae86 | ||
|
|
77057eb829 | ||
|
|
2b1a7b840d | ||
|
|
e07db88bc0 | ||
|
|
c2282b0e73 | ||
|
|
593bf09d8d | ||
|
|
534ed77ebf | ||
|
|
193299988d | ||
|
|
258e83c904 | ||
|
|
2177a2b805 | ||
|
|
68164415ce | ||
|
|
7646599b66 | ||
|
|
cccd82a617 | ||
|
|
694922f627 | ||
|
|
e8faf28e6a |
12
CHANGELOG.md
12
CHANGELOG.md
@@ -11,19 +11,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
- Updated `daily-python` to 0.19.6.
|
||||
|
||||
- Changed `TavusVideoService` to send audio or video frames only after the
|
||||
transport is ready, preventing warning messages at startup.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed an issue in `LiveKitTransport` where empty `AudioRawFrame`s were pushed
|
||||
down the pipeline. This resulted in warnings by the STT processor.
|
||||
- Fixed `PiperTTSService` to send text as a JSON object in the request body,
|
||||
resolving compatibility with Piper's HTTP API.
|
||||
|
||||
- Fixed an issue with the `TavusVideoService` where an error was thrown due to
|
||||
missing transcription callbacks.
|
||||
|
||||
- Fixed an issue in `SpeechmaticsSTTService` where the `user_id` was set to
|
||||
`None` when diarization is not enabled.
|
||||
|
||||
### Performance
|
||||
|
||||
- Fixed an issue in `TaskObserver` (a proxy to all observers) that was degrading
|
||||
global performance.
|
||||
|
||||
### Other
|
||||
|
||||
- Allow Daily transport to quickstart bot example.
|
||||
|
||||
## [0.0.77] - 2025-07-31
|
||||
|
||||
### Added
|
||||
|
||||
@@ -49,7 +49,7 @@ async def run_bot(transport: BaseTransport):
|
||||
This example will use diarization within our STT service and output the words spoken by
|
||||
each individual speaker and wrap them with XML tags.
|
||||
|
||||
If you do not wish to use diarization, then set the `enable_speaker_diarization` parameter
|
||||
If you do not wish to use diarization, then set the `enable_diarization` parameter
|
||||
to `False` or omit it altogether. The `text_format` will only be used if diarization is enabled.
|
||||
|
||||
By default, this example will use our ENHANCED operating point, which is optimized for
|
||||
|
||||
@@ -4,6 +4,11 @@
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
# /// script
|
||||
# dependencies = [
|
||||
# "pipecat-ai[daily,webrtc,websocket,runner,silero,deepgram,google,cartesia,heygen]>=0.0.77",
|
||||
# ]
|
||||
# ///
|
||||
|
||||
import os
|
||||
|
||||
|
||||
@@ -32,12 +32,13 @@ from pipecat.pipeline.runner import PipelineRunner
|
||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||
from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor
|
||||
from pipecat.runner.types import RunnerArguments
|
||||
from pipecat.runner.types import DailyRunnerArguments, RunnerArguments, SmallWebRTCRunnerArguments
|
||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||
from pipecat.services.openai.llm import OpenAILLMService
|
||||
from pipecat.transports.base_transport import BaseTransport, TransportParams
|
||||
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
||||
|
||||
load_dotenv(override=True)
|
||||
|
||||
@@ -108,16 +109,33 @@ async def run_bot(transport: BaseTransport):
|
||||
async def bot(runner_args: RunnerArguments):
|
||||
"""Main bot entry point for the bot starter."""
|
||||
|
||||
transport = SmallWebRTCTransport(
|
||||
params=TransportParams(
|
||||
audio_in_enabled=True,
|
||||
audio_out_enabled=True,
|
||||
vad_analyzer=SileroVADAnalyzer(),
|
||||
),
|
||||
webrtc_connection=runner_args.webrtc_connection,
|
||||
)
|
||||
transport = None
|
||||
|
||||
await run_bot(transport)
|
||||
if isinstance(runner_args, SmallWebRTCRunnerArguments):
|
||||
transport = SmallWebRTCTransport(
|
||||
params=TransportParams(
|
||||
audio_in_enabled=True,
|
||||
audio_out_enabled=True,
|
||||
vad_analyzer=SileroVADAnalyzer(),
|
||||
),
|
||||
webrtc_connection=runner_args.webrtc_connection,
|
||||
)
|
||||
elif isinstance(runner_args, DailyRunnerArguments):
|
||||
transport = DailyTransport(
|
||||
runner_args.room_url,
|
||||
runner_args.token,
|
||||
"Pipecat Quickstart",
|
||||
params=DailyParams(
|
||||
audio_in_enabled=True,
|
||||
audio_out_enabled=True,
|
||||
vad_analyzer=SileroVADAnalyzer(),
|
||||
),
|
||||
)
|
||||
else:
|
||||
logger.warning("Unsupported transport")
|
||||
|
||||
if transport:
|
||||
await run_bot(transport)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -983,17 +983,14 @@ class GeminiMultimodalLiveLLMService(LLMService):
|
||||
with audio and video inputs, preventing temporal misalignment that can occur
|
||||
when different modalities are processed through separate API pathways.
|
||||
|
||||
After sending the text, we signal turn completion to trigger a model response
|
||||
for text-only interactions.
|
||||
For realtimeInput, turn completion is automatically inferred by the API based
|
||||
on user activity, so no explicit turnComplete signal is needed.
|
||||
|
||||
Args:
|
||||
text: The text to send as user input.
|
||||
"""
|
||||
evt = events.TextInputMessage.from_text(text)
|
||||
await self.send_client_event(evt)
|
||||
# After sending text, we need to signal that the turn is complete.
|
||||
evt = events.ClientContentMessage.model_validate({"clientContent": {"turnComplete": True}})
|
||||
await self.send_client_event(evt)
|
||||
|
||||
async def _send_user_video(self, frame):
|
||||
"""Send user video frame to Gemini Live API."""
|
||||
|
||||
@@ -84,7 +84,9 @@ class PiperTTSService(TTSService):
|
||||
try:
|
||||
await self.start_ttfb_metrics()
|
||||
|
||||
async with self._session.post(self._base_url, json=text, headers=headers) as response:
|
||||
async with self._session.post(
|
||||
self._base_url, json={"text": text}, headers=headers
|
||||
) as response:
|
||||
if response.status != 200:
|
||||
error = await response.text()
|
||||
logger.error(
|
||||
|
||||
@@ -191,7 +191,7 @@ class SpeakerFragments:
|
||||
passive_format = active_format
|
||||
return {
|
||||
"text": self._format_text(active_format if self.is_active else passive_format),
|
||||
"user_id": self.speaker_id,
|
||||
"user_id": self.speaker_id or "",
|
||||
"timestamp": self.timestamp,
|
||||
"language": self.language,
|
||||
"result": [frag.result for frag in self.fragments],
|
||||
|
||||
@@ -24,6 +24,7 @@ from pipecat.frames.frames import (
|
||||
Frame,
|
||||
OutputAudioRawFrame,
|
||||
OutputImageRawFrame,
|
||||
OutputTransportReadyFrame,
|
||||
StartFrame,
|
||||
StartInterruptionFrame,
|
||||
TTSAudioRawFrame,
|
||||
@@ -81,6 +82,7 @@ class TavusVideoService(AIService):
|
||||
self._send_task: Optional[asyncio.Task] = None
|
||||
# This is the custom track destination expected by Tavus
|
||||
self._transport_destination: Optional[str] = "stream"
|
||||
self._transport_ready = False
|
||||
|
||||
async def setup(self, setup: FrameProcessorSetup):
|
||||
"""Set up the Tavus video service.
|
||||
@@ -145,7 +147,8 @@ class TavusVideoService(AIService):
|
||||
format=video_frame.color_format,
|
||||
)
|
||||
frame.transport_source = video_source
|
||||
await self.push_frame(frame)
|
||||
if self._transport_ready:
|
||||
await self.push_frame(frame)
|
||||
|
||||
async def _on_participant_audio_data(
|
||||
self, participant_id: str, audio: AudioData, audio_source: str
|
||||
@@ -157,7 +160,8 @@ class TavusVideoService(AIService):
|
||||
num_channels=audio.num_channels,
|
||||
)
|
||||
frame.transport_source = audio_source
|
||||
await self.push_frame(frame)
|
||||
if self._transport_ready:
|
||||
await self.push_frame(frame)
|
||||
|
||||
def can_generate_metrics(self) -> bool:
|
||||
"""Check if this service can generate processing metrics.
|
||||
@@ -221,6 +225,9 @@ class TavusVideoService(AIService):
|
||||
await self.push_frame(frame, direction)
|
||||
elif isinstance(frame, TTSAudioRawFrame):
|
||||
await self._handle_audio_frame(frame)
|
||||
elif isinstance(frame, OutputTransportReadyFrame):
|
||||
self._transport_ready = True
|
||||
await self.push_frame(frame, direction)
|
||||
else:
|
||||
await self.push_frame(frame, direction)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user