Compare commits

...

21 Commits

Author SHA1 Message Date
Aleix Conchillo Flaqué
6e518cc704 examples(quickstart): allow Daily transport 2025-08-04 09:32:15 -07:00
Vanessa Pyne
96b1000e52 Merge pull request #2341 from getchannel/realtime-text
Hotfix: Correct Gemini Live API class to fix 1007 payload error.
2025-08-04 11:03:57 -05:00
Filipi da Silva Fuchter
0184a8c231 Merge pull request #2351 from pipecat-ai/filipi/tavus_transport_ready
Changed `TavusVideoService` to send audio or video frames only after the   transport is ready, preventing warning messages at startup.
2025-08-04 11:48:27 -03:00
Filipi Fuchter
c22866ed58 Mentioning the TavusVideoService fix in the changelog. 2025-08-04 11:46:24 -03:00
Filipi Fuchter
0e533d21be Only send audio and video from the tavus video service if the transport is ready. 2025-08-04 10:52:30 -03:00
Mark Backman
6f6f4c3dea Merge pull request #2348 from sam-s10s/speechmatics-stt
Fix for Speechmatics STT
2025-08-04 06:15:39 -07:00
Mark Backman
54ff10ae86 Merge pull request #2332 from hankehly/fix-piper-tts-json-payload
Fix PiperTTSService to send TTS input as JSON object
2025-08-03 17:39:04 -07:00
hankehly
77057eb829 Fix ruff formatting 2025-08-04 08:13:16 +09:00
Mark Backman
2b1a7b840d Merge pull request #2346 from adenta/heygen-testing
me trying to get heygen working
2025-08-03 14:11:14 -07:00
Sam Sykes
e07db88bc0 Updated changelog. 2025-08-03 22:11:10 +01:00
Sam Sykes
c2282b0e73 Set user_id to "" (not None) for RTVIProcessor. 2025-08-03 22:08:22 +01:00
Andrew Denta
593bf09d8d update script 2025-08-03 17:01:27 -04:00
Sam Sykes
534ed77ebf Merge branch 'main' into speechmatics-stt 2025-08-03 21:51:35 +01:00
Andrew Denta
193299988d me trying to get heygen working 2025-08-03 13:42:07 -04:00
Pete
258e83c904 Fix: Correct Gemini Live API text input to prevent 1007 WebSocket errors
- Restore TextInputMessage.realtimeInput structure for correct API format
- Remove invalid turnComplete message from _send_user_text method
- turnComplete is only valid for clientContent, not realtimeInput messages
- realtimeInput text completion is automatically inferred by the API

This fixes WebSocket 1007 errors caused by mixing realtimeInput and
clientContent message types in violation of the Gemini Live API contract.
2025-08-03 10:58:59 -04:00
hankehly
2177a2b805 Remove trailing space 2025-08-03 10:34:27 +09:00
hankehly
68164415ce Format changelog entry 2025-08-03 10:26:37 +09:00
hankehly
7646599b66 Add changelog entry 2025-08-03 10:23:58 +09:00
Pete
cccd82a617 Refactor TextInputMessage class to replace realtimeInput with a text attribute.
This was sending a 1007 because it was wrapping RealtimeInput in the json.

- Updated the `TextInputMessage` class to directly store text input as a string.
- Modified the `from_text` class method to create an instance using the new `text` attribute.
2025-08-02 14:34:00 -04:00
hankehly
694922f627 Fix PiperTTSService to send TTS input as JSON object 2025-08-02 15:29:16 +09:00
Sam Sykes
e8faf28e6a Doc fix for incorrect argument name. 2025-07-31 22:30:54 +01:00
8 changed files with 61 additions and 20 deletions

View File

@@ -11,19 +11,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Updated `daily-python` to 0.19.6.
- Changed `TavusVideoService` to send audio or video frames only after the
transport is ready, preventing warning messages at startup.
### Fixed
- Fixed an issue in `LiveKitTransport` where empty `AudioRawFrame`s were pushed
down the pipeline. This resulted in warnings by the STT processor.
- Fixed `PiperTTSService` to send text as a JSON object in the request body,
resolving compatibility with Piper's HTTP API.
- Fixed an issue with the `TavusVideoService` where an error was thrown due to
missing transcription callbacks.
- Fixed an issue in `SpeechmaticsSTTService` where the `user_id` was set to
`None` when diarization is not enabled.
### Performance
- Fixed an issue in `TaskObserver` (a proxy to all observers) that was degrading
global performance.
### Other
- Allow Daily transport to quickstart bot example.
## [0.0.77] - 2025-07-31
### Added

View File

@@ -49,7 +49,7 @@ async def run_bot(transport: BaseTransport):
This example will use diarization within our STT service and output the words spoken by
each individual speaker and wrap them with XML tags.
If you do not wish to use diarization, then set the `enable_speaker_diarization` parameter
If you do not wish to use diarization, then set the `enable_diarization` parameter
to `False` or omit it altogether. The `text_format` will only be used if diarization is enabled.
By default, this example will use our ENHANCED operating point, which is optimized for

View File

@@ -4,6 +4,11 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
# /// script
# dependencies = [
# "pipecat-ai[daily,webrtc,websocket,runner,silero,deepgram,google,cartesia,heygen]>=0.0.77",
# ]
# ///
import os

View File

@@ -32,12 +32,13 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor
from pipecat.runner.types import RunnerArguments
from pipecat.runner.types import DailyRunnerArguments, RunnerArguments, SmallWebRTCRunnerArguments
from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.services.daily import DailyParams, DailyTransport
load_dotenv(override=True)
@@ -108,16 +109,33 @@ async def run_bot(transport: BaseTransport):
async def bot(runner_args: RunnerArguments):
"""Main bot entry point for the bot starter."""
transport = SmallWebRTCTransport(
params=TransportParams(
audio_in_enabled=True,
audio_out_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
),
webrtc_connection=runner_args.webrtc_connection,
)
transport = None
await run_bot(transport)
if isinstance(runner_args, SmallWebRTCRunnerArguments):
transport = SmallWebRTCTransport(
params=TransportParams(
audio_in_enabled=True,
audio_out_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
),
webrtc_connection=runner_args.webrtc_connection,
)
elif isinstance(runner_args, DailyRunnerArguments):
transport = DailyTransport(
runner_args.room_url,
runner_args.token,
"Pipecat Quickstart",
params=DailyParams(
audio_in_enabled=True,
audio_out_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
),
)
else:
logger.warning("Unsupported transport")
if transport:
await run_bot(transport)
if __name__ == "__main__":

View File

@@ -983,17 +983,14 @@ class GeminiMultimodalLiveLLMService(LLMService):
with audio and video inputs, preventing temporal misalignment that can occur
when different modalities are processed through separate API pathways.
After sending the text, we signal turn completion to trigger a model response
for text-only interactions.
For realtimeInput, turn completion is automatically inferred by the API based
on user activity, so no explicit turnComplete signal is needed.
Args:
text: The text to send as user input.
"""
evt = events.TextInputMessage.from_text(text)
await self.send_client_event(evt)
# After sending text, we need to signal that the turn is complete.
evt = events.ClientContentMessage.model_validate({"clientContent": {"turnComplete": True}})
await self.send_client_event(evt)
async def _send_user_video(self, frame):
"""Send user video frame to Gemini Live API."""

View File

@@ -84,7 +84,9 @@ class PiperTTSService(TTSService):
try:
await self.start_ttfb_metrics()
async with self._session.post(self._base_url, json=text, headers=headers) as response:
async with self._session.post(
self._base_url, json={"text": text}, headers=headers
) as response:
if response.status != 200:
error = await response.text()
logger.error(

View File

@@ -191,7 +191,7 @@ class SpeakerFragments:
passive_format = active_format
return {
"text": self._format_text(active_format if self.is_active else passive_format),
"user_id": self.speaker_id,
"user_id": self.speaker_id or "",
"timestamp": self.timestamp,
"language": self.language,
"result": [frag.result for frag in self.fragments],

View File

@@ -24,6 +24,7 @@ from pipecat.frames.frames import (
Frame,
OutputAudioRawFrame,
OutputImageRawFrame,
OutputTransportReadyFrame,
StartFrame,
StartInterruptionFrame,
TTSAudioRawFrame,
@@ -81,6 +82,7 @@ class TavusVideoService(AIService):
self._send_task: Optional[asyncio.Task] = None
# This is the custom track destination expected by Tavus
self._transport_destination: Optional[str] = "stream"
self._transport_ready = False
async def setup(self, setup: FrameProcessorSetup):
"""Set up the Tavus video service.
@@ -145,7 +147,8 @@ class TavusVideoService(AIService):
format=video_frame.color_format,
)
frame.transport_source = video_source
await self.push_frame(frame)
if self._transport_ready:
await self.push_frame(frame)
async def _on_participant_audio_data(
self, participant_id: str, audio: AudioData, audio_source: str
@@ -157,7 +160,8 @@ class TavusVideoService(AIService):
num_channels=audio.num_channels,
)
frame.transport_source = audio_source
await self.push_frame(frame)
if self._transport_ready:
await self.push_frame(frame)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -221,6 +225,9 @@ class TavusVideoService(AIService):
await self.push_frame(frame, direction)
elif isinstance(frame, TTSAudioRawFrame):
await self._handle_audio_frame(frame)
elif isinstance(frame, OutputTransportReadyFrame):
self._transport_ready = True
await self.push_frame(frame, direction)
else:
await self.push_frame(frame, direction)