From 03eb22fe0a465472406ad4c241352a20c380acfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 10 Jun 2025 09:05:58 -0700 Subject: [PATCH 01/19] examples(22b): remove unnecessary parallel pipeline branch --- .../foundational/22b-natural-conversation-proposal.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/examples/foundational/22b-natural-conversation-proposal.py b/examples/foundational/22b-natural-conversation-proposal.py index 1b03f341e..b76bfd274 100644 --- a/examples/foundational/22b-natural-conversation-proposal.py +++ b/examples/foundational/22b-natural-conversation-proposal.py @@ -312,9 +312,6 @@ async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_si # to start the conversation. bot_output_gate = OutputGate(notifier=notifier, start_open=True) - async def block_user_stopped_speaking(frame): - return not isinstance(frame, UserStoppedSpeakingFrame) - async def pass_only_llm_trigger_frames(frame): return ( isinstance(frame, OpenAILLMContextFrame) @@ -331,11 +328,6 @@ async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_si stt, context_aggregator.user(), ParallelPipeline( - [ - # Pass everything except UserStoppedSpeaking to the elements after - # this ParallelPipeline - FunctionFilter(filter=block_user_stopped_speaking), - ], [ # Ignore everything except an OpenAILLMContextFrame. Pass a specially constructed # LLMMessagesFrame to the statement classifier LLM. The only frame this From cd98657e3c9d266b035dfa1f374e9e1b46dc737c Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 10 Jun 2025 15:09:13 -0400 Subject: [PATCH 02/19] Add Cartesia STT docs link to README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f906cb8bb..7ec4c6000 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ You can connect to Pipecat from any platform using our official SDKs: | Category | Services | | ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), Cartesia, [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [Parakeet (NVIDIA)](https://docs.pipecat.ai/server/services/stt/parakeet), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [Parakeet (NVIDIA)](https://docs.pipecat.ai/server/services/stt/parakeet), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | | LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | | Text-to-Speech | [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [FastPitch (NVIDIA)](https://docs.pipecat.ai/server/services/tts/fastpitch), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | | Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | From 257dbe3104e9939d36712b7e5dbeb763ba319933 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 10 Jun 2025 15:14:47 -0400 Subject: [PATCH 03/19] Fix model param error --- src/pipecat/services/cartesia/stt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/cartesia/stt.py b/src/pipecat/services/cartesia/stt.py index 618908ed9..104e4b2c5 100644 --- a/src/pipecat/services/cartesia/stt.py +++ b/src/pipecat/services/cartesia/stt.py @@ -101,7 +101,7 @@ class CartesiaSTTService(STTService): ) self._settings = merged_options - self.set_model_name(merged_options["model"]) + self.set_model_name(merged_options.model) self._api_key = api_key self._base_url = base_url or "api.cartesia.ai" self._connection = None From c101c9c8e15eb1295fa3654a7df0622d21a12614 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 10 Jun 2025 13:37:28 -0700 Subject: [PATCH 04/19] update CHANGELOG for 0.0.70 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fc5abde4..1537fd83b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.0.70] ### Added From c59180dd6eb7fb8e3c9708107ce074db16231911 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 10 Jun 2025 14:23:02 -0700 Subject: [PATCH 05/19] udpate CHANGELOG --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1537fd83b..54bb45013 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,13 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.0.70] +## [Unreleased] + +### Fixed + +- Fixed an issue with `CartesiaSTTService` initialization. + +## [0.0.70] - 2025-06-10 ### Added From d3df75aaa067088cfebd27535bdc04d1de577858 Mon Sep 17 00:00:00 2001 From: Sunah Suh Date: Tue, 10 Jun 2025 16:32:24 -0500 Subject: [PATCH 06/19] =?UTF-8?q?Add=20`additional=5Fspan=5Fattributes`=20?= =?UTF-8?q?param=20to=20PipelineTask=20for=20extra=20otel=E2=80=A6=20(#199?= =?UTF-8?q?2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 5 +++++ src/pipecat/pipeline/task.py | 10 ++++++++-- src/pipecat/utils/tracing/turn_trace_observer.py | 10 +++++++++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54bb45013..be7440cf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Adds a parameter called `additional_span_attributes` to PipelineTask that + lets you add any additional attributes you'd like to the conversation span. + ### Fixed - Fixed an issue with `CartesiaSTTService` initialization. diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index 6bf5dd688..736f98244 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -184,7 +184,9 @@ class PipelineTask(BaseTask): the idle timeout is reached. enable_turn_tracking: Whether to enable turn tracking. enable_turn_tracing: Whether to enable turn tracing. - + conversation_id: Optional custom ID for the conversation. + additional_span_attributes: Optional dictionary of attributes to propagate as + OpenTelemetry conversation span attributes. """ def __init__( @@ -205,6 +207,7 @@ class PipelineTask(BaseTask): enable_turn_tracking: bool = True, enable_tracing: bool = False, conversation_id: Optional[str] = None, + additional_span_attributes: Optional[dict] = None, ): super().__init__() self._pipeline = pipeline @@ -217,6 +220,7 @@ class PipelineTask(BaseTask): self._enable_turn_tracking = enable_turn_tracking self._enable_tracing = enable_tracing and is_tracing_available() self._conversation_id = conversation_id + self._additional_span_attributes = additional_span_attributes or {} if self._params.observers: import warnings @@ -235,7 +239,9 @@ class PipelineTask(BaseTask): observers.append(self._turn_tracking_observer) if self._enable_tracing and self._turn_tracking_observer: self._turn_trace_observer = TurnTraceObserver( - self._turn_tracking_observer, conversation_id=self._conversation_id + self._turn_tracking_observer, + conversation_id=self._conversation_id, + additional_span_attributes=self._additional_span_attributes, ) observers.append(self._turn_trace_observer) self._finished = False diff --git a/src/pipecat/utils/tracing/turn_trace_observer.py b/src/pipecat/utils/tracing/turn_trace_observer.py index 26fc5b38a..f67ca3b28 100644 --- a/src/pipecat/utils/tracing/turn_trace_observer.py +++ b/src/pipecat/utils/tracing/turn_trace_observer.py @@ -35,7 +35,11 @@ class TurnTraceObserver(BaseObserver): """ def __init__( - self, turn_tracker: TurnTrackingObserver, conversation_id: Optional[str] = None, **kwargs + self, + turn_tracker: TurnTrackingObserver, + conversation_id: Optional[str] = None, + additional_span_attributes: Optional[dict] = None, + **kwargs, ): super().__init__(**kwargs) self._turn_tracker = turn_tracker @@ -47,6 +51,7 @@ class TurnTraceObserver(BaseObserver): # Conversation tracking properties self._conversation_span: Optional["Span"] = None self._conversation_id = conversation_id + self._additional_span_attributes = additional_span_attributes or {} if turn_tracker: @@ -89,6 +94,9 @@ class TurnTraceObserver(BaseObserver): # Set span attributes self._conversation_span.set_attribute("conversation.id", conversation_id) self._conversation_span.set_attribute("conversation.type", "voice") + # Set custom otel attributes if provided + for k, v in (self._additional_span_attributes or {}).items(): + self._conversation_span.set_attribute(k, v) # Update the conversation context provider context_provider.set_current_conversation_context( From 61a5154e49f948adeb1bd087a80545ec97f6966a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 10 Jun 2025 14:34:30 -0700 Subject: [PATCH 07/19] update CHANGELOG for 0.0.71 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be7440cf2..41d1b6236 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.0.71] - 2025-06-10 ### Added From c1db13ceebd7f5e5d180325505cf3f6ae818f1f7 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Thu, 12 Jun 2025 12:07:33 -0300 Subject: [PATCH 08/19] Fixed an issue with `GoogleSTTService` where it was constantly reconnecting before starting to receive audio from the user. --- CHANGELOG.md | 7 +++++++ src/pipecat/services/google/stt.py | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41d1b6236..3ee2453e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Fixed + +- Fixed an issue with `GoogleSTTService` where it was constantly reconnecting + before starting to receive audio from the user. + ## [0.0.71] - 2025-06-10 ### Added diff --git a/src/pipecat/services/google/stt.py b/src/pipecat/services/google/stt.py index 4fd129af3..bf60541f5 100644 --- a/src/pipecat/services/google/stt.py +++ b/src/pipecat/services/google/stt.py @@ -747,6 +747,11 @@ class GoogleSTTService(STTService): try: while True: try: + if self._request_queue.empty(): + # wait for 10ms in case we don't have audio + await asyncio.sleep(0.01) + continue + # Start bi-directional streaming streaming_recognize = await self._client.streaming_recognize( requests=self._request_generator() From 69c63293fb7f83ddb1b477adb7463112ad16d8a2 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 12 Jun 2025 11:43:27 -0400 Subject: [PATCH 09/19] fix: GoogleLLMService TTFB value --- CHANGELOG.md | 6 ++++++ src/pipecat/services/google/llm.py | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41d1b6236..9c38dcd55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Fixed + +- Fixed an issue where `GoogleLLMService`'s TTFB value was incorrect. + ## [0.0.71] - 2025-06-10 ### Added diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index 8960bda31..f983b7342 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -555,10 +555,11 @@ class GoogleLLMService(LLMService): contents=messages, config=generation_config, ) - await self.stop_ttfb_metrics() function_calls = [] async for chunk in response: + # Stop TTFB metrics after the first chunk + await self.stop_ttfb_metrics() if chunk.usage_metadata: prompt_tokens += chunk.usage_metadata.prompt_token_count or 0 completion_tokens += chunk.usage_metadata.candidates_token_count or 0 From 22f4f0b79e9006c8f6971ae0c2b05dbd91c87524 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 12 Jun 2025 11:45:59 -0400 Subject: [PATCH 10/19] Update 14e example name --- CHANGELOG.md | 4 ++++ ...ction-calling-gemini.py => 14e-function-calling-google.py} | 0 2 files changed, 4 insertions(+) rename examples/foundational/{14e-function-calling-gemini.py => 14e-function-calling-google.py} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c38dcd55..f6a009fd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed an issue where `GoogleLLMService`'s TTFB value was incorrect. +### Other + +- Rename `14e-function-calling-gemini.py` to `14e-function-calling-google.py`. + ## [0.0.71] - 2025-06-10 ### Added diff --git a/examples/foundational/14e-function-calling-gemini.py b/examples/foundational/14e-function-calling-google.py similarity index 100% rename from examples/foundational/14e-function-calling-gemini.py rename to examples/foundational/14e-function-calling-google.py From 1e3fa4a9c7fa94aa4957909dffc0d9c15d5a4b9d Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sat, 14 Jun 2025 17:41:44 -0400 Subject: [PATCH 11/19] fix groq wav file header parsing --- src/pipecat/services/groq/tts.py | 38 ++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/src/pipecat/services/groq/tts.py b/src/pipecat/services/groq/tts.py index 6f73b1629..33fd3ce34 100644 --- a/src/pipecat/services/groq/tts.py +++ b/src/pipecat/services/groq/tts.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +import io +import wave from typing import AsyncGenerator, Optional from loguru import logger @@ -78,22 +80,26 @@ class GroqTTSService(TTSService): await self.start_ttfb_metrics() yield TTSStartedFrame() - response = await self._client.audio.speech.create( - model=self._model_name, - voice=self._voice_id, - response_format=self._output_format, - input=text, - ) + try: + response = await self._client.audio.speech.create( + model=self._model_name, + voice=self._voice_id, + response_format=self._output_format, + input=text, + ) - async for data in response.iter_bytes(): - if measuring_ttfb: - await self.stop_ttfb_metrics() - measuring_ttfb = False - # remove wav header if present - if data.startswith(b"RIFF"): - data = data[44:] - if len(data) == 0: - continue - yield TTSAudioRawFrame(data, self.sample_rate, 1) + async for data in response.iter_bytes(): + if measuring_ttfb: + await self.stop_ttfb_metrics() + measuring_ttfb = False + + with wave.open(io.BytesIO(data)) as w: + channels = w.getnchannels() + frame_rate = w.getframerate() + num_frames = w.getnframes() + bytes = w.readframes(num_frames) + yield TTSAudioRawFrame(bytes, frame_rate, channels) + except Exception as e: + logger.error(f"{self} exception: {e}") yield TTSStoppedFrame() From fe16ed3c73882c42e1cbbc1020757cad15112bc0 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sun, 15 Jun 2025 10:49:40 -0700 Subject: [PATCH 12/19] added changelog entry --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c8648272..cfbeb5010 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an issue with `GroqTTSService` where it was not properly parsing the + WAV file header. + - Fixed an issue with `GoogleSTTService` where it was constantly reconnecting before starting to receive audio from the user. From e2c15169b8fa96430aaef351cbd2642c431fa832 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20W=C5=82odarczyk?= <49692261+wuodar@users.noreply.github.com> Date: Sun, 15 Jun 2025 21:44:06 +0200 Subject: [PATCH 13/19] feat: support polish language in Amazon Transcribe --- src/pipecat/services/aws/stt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pipecat/services/aws/stt.py b/src/pipecat/services/aws/stt.py index 4490f3795..d6aa1fa5d 100644 --- a/src/pipecat/services/aws/stt.py +++ b/src/pipecat/services/aws/stt.py @@ -266,6 +266,7 @@ class AWSTranscribeSTTService(STTService): Language.JA: "ja-JP", Language.KO: "ko-KR", Language.ZH: "zh-CN", + Language.PL: "pl-PL", } return language_map.get(language) From a4ea0d2b8238f2059a9e74ddaed2013fe9e78ab1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Sun, 15 Jun 2025 21:05:03 -0700 Subject: [PATCH 14/19] dev-requirements: update pyright 1.1.400 and ruff 0.11.13 --- dev-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index af1c35721..2bab71684 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -3,11 +3,11 @@ coverage~=7.6.12 grpcio-tools~=1.67.1 pip-tools~=7.4.1 pre-commit~=4.0.1 -pyright~=1.1.397 +pyright~=1.1.400 pytest~=8.3.4 pytest-asyncio~=0.25.3 pytest-aiohttp==1.1.0 -ruff~=0.11.1 +ruff~=0.11.13 setuptools~=70.0.0 setuptools_scm~=8.1.0 python-dotenv~=1.0.1 From d1bee22d73213f0ac282c5d8046dd049534fcf19 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 13 Jun 2025 17:21:29 -0400 Subject: [PATCH 15/19] Expose has_function_calls_in_progress property --- CHANGELOG.md | 10 +++++++++- src/pipecat/processors/aggregators/llm_response.py | 9 +++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfbeb5010..618ef2faa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,12 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Added a property called `has_function_calls_in_progress` in + `LLMAssistantContextAggregator` that exposes whether a function call is in + progress. + ### Fixed - Fixed an issue with `GroqTTSService` where it was not properly parsing the WAV file header. -- Fixed an issue with `GoogleSTTService` where it was constantly reconnecting +- Fixed an issue with `GoogleSTTService` where it was constantly reconnecting + +- Fixed an issue with `GoogleSTTService` where it was constantly reconnecting before starting to receive audio from the user. - Fixed an issue where `GoogleLLMService`'s TTFB value was incorrect. diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 479199550..e9aebb2a0 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -504,6 +504,15 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): self._function_calls_in_progress: Dict[str, Optional[FunctionCallInProgressFrame]] = {} self._context_updated_tasks: Set[asyncio.Task] = set() + @property + def has_function_calls_in_progress(self) -> bool: + """Check if there are any function calls currently in progress. + + Returns: + bool: True if function calls are in progress, False otherwise + """ + return bool(self._function_calls_in_progress) + async def handle_aggregation(self, aggregation: str): self._context.add_message({"role": "assistant", "content": aggregation}) From 14dc6a79843e38fd418e67ef07f094b2527c1b19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Sun, 15 Jun 2025 22:38:07 -0700 Subject: [PATCH 16/19] FrameProcessor: handle new FrameProcessorPauseFrame/FrameProcessorResumeFrame --- CHANGELOG.md | 12 +++++-- src/pipecat/frames/frames.py | 44 +++++++++++++++++++++++ src/pipecat/processors/frame_processor.py | 16 +++++++++ 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 618ef2faa..94e1db15c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added new frames `FrameProcessorPauseFrame` and `FrameProcessorResumeFrame` + which allow pausing and resuming frame processing for a given frame + processor. These are control frames, so they are ordered. Pausing frame + processor will keep old frames in the internal queues until resume takes + place. Frames being pushed while a frame processor is paused will be pushed to + the queues. When frame processing is resumed all queued frames will be + processed in order. Also added `FrameProcessorPauseUrgentFrame` and + `FrameProcessorResumeUrgentFrame` which are system frames and therefore they + have high priority. + - Added a property called `has_function_calls_in_progress` in `LLMAssistantContextAggregator` that exposes whether a function call is in progress. @@ -18,8 +28,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed an issue with `GroqTTSService` where it was not properly parsing the WAV file header. -- Fixed an issue with `GoogleSTTService` where it was constantly reconnecting - - Fixed an issue with `GoogleSTTService` where it was constantly reconnecting before starting to receive audio from the user. diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 63caf9a09..ec368789d 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -527,6 +527,29 @@ class StopTaskFrame(SystemFrame): pass +@dataclass +class FrameProcessorPauseUrgentFrame(SystemFrame): + """This processor is used to pause frame processing for the given processor + as fast as possible. Pausing frame processing will keep frames in the + internal queue which will then be processed when frame processing is resumed + with `FrameProcessorResumeFrame`. + + """ + + processor: str + + +@dataclass +class FrameProcessorResumeUrgentFrame(SystemFrame): + """This processor is used to resume frame processing for the given processor + if it was previously paused as fast as possible. After resuming frame + processing all queued frames will be processed in the order received. + + """ + + processor: str + + @dataclass class StartInterruptionFrame(SystemFrame): """Emitted by VAD to indicate that a user has started speaking (i.e. is @@ -854,6 +877,27 @@ class StopFrame(ControlFrame): pass +@dataclass +class FrameProcessorPauseFrame(ControlFrame): + """This processor is used to pause frame processing for the given + processor. Pausing frame processing will keep frames in the internal queue + which will then be processed when frame processing is resumed with + `FrameProcessorResumeFrame`.""" + + processor: str + + +@dataclass +class FrameProcessorResumeFrame(ControlFrame): + """This processor is used to resume frame processing for the given processor + if it was previously paused. After resuming frame processing all queued + frames will be processed in the order received. + + """ + + processor: str + + @dataclass class LLMFullResponseStartFrame(ControlFrame): """Used to indicate the beginning of an LLM response. Following by one or diff --git a/src/pipecat/processors/frame_processor.py b/src/pipecat/processors/frame_processor.py index 3b66973dd..1d2f066ed 100644 --- a/src/pipecat/processors/frame_processor.py +++ b/src/pipecat/processors/frame_processor.py @@ -17,6 +17,10 @@ from pipecat.frames.frames import ( CancelFrame, ErrorFrame, Frame, + FrameProcessorPauseFrame, + FrameProcessorPauseUrgentFrame, + FrameProcessorResumeFrame, + FrameProcessorResumeUrgentFrame, StartFrame, StartInterruptionFrame, StopInterruptionFrame, @@ -259,6 +263,10 @@ class FrameProcessor(BaseObject): self._should_report_ttfb = True elif isinstance(frame, CancelFrame): await self.__cancel(frame) + elif isinstance(frame, (FrameProcessorPauseFrame, FrameProcessorPauseUrgentFrame)): + await self.__pause(frame) + elif isinstance(frame, (FrameProcessorResumeFrame, FrameProcessorResumeUrgentFrame)): + await self.__resume(frame) async def push_error(self, error: ErrorFrame): await self.push_frame(error, FrameDirection.UPSTREAM) @@ -287,6 +295,14 @@ class FrameProcessor(BaseObject): await self.__cancel_input_task() await self.__cancel_push_task() + async def __pause(self, frame: FrameProcessorPauseFrame | FrameProcessorPauseUrgentFrame): + if frame.name == self.name: + await self.pause_processing_frames() + + async def __resume(self, frame: FrameProcessorResumeFrame | FrameProcessorResumeUrgentFrame): + if frame.name == self.name: + await self.resume_processing_frames() + # # Handle interruptions # From 20eebb08e9f059e0800ef3f40429f904becd79d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 16 Jun 2025 10:34:56 -0700 Subject: [PATCH 17/19] update CHANGELOG with AWSTranscribeSTTService Polish support --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94e1db15c..d8df25cf9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added Polish support to `AWSTranscribeSTTService`. + - Added new frames `FrameProcessorPauseFrame` and `FrameProcessorResumeFrame` which allow pausing and resuming frame processing for a given frame processor. These are control frames, so they are ordered. Pausing frame From 7ddc706434b42019434e8d353583b573f71a67d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 17 Jun 2025 09:30:28 -0700 Subject: [PATCH 18/19] update daily-python to 0.19.3 --- CHANGELOG.md | 4 ++++ pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8df25cf9..59efa9e3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `LLMAssistantContextAggregator` that exposes whether a function call is in progress. +### Changed + +- Upgraded `daily-python` to 0.19.3. + ### Fixed - Fixed an issue with `GroqTTSService` where it was not properly parsing the diff --git a/pyproject.toml b/pyproject.toml index 4652b684a..0c98d6226 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ azure = [ "azure-cognitiveservices-speech~=1.42.0"] cartesia = [ "cartesia~=2.0.3", "websockets~=13.1" ] cerebras = [] deepseek = [] -daily = [ "daily-python~=0.19.2" ] +daily = [ "daily-python~=0.19.3" ] deepgram = [ "deepgram-sdk~=4.1.0" ] elevenlabs = [ "websockets~=13.1" ] fal = [ "fal-client~=0.5.9" ] From 11b6e409bb2309eed5e739d1e70f83544109e9c3 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Tue, 17 Jun 2025 15:22:31 -0300 Subject: [PATCH 19/19] Bumping pipecat-ai-krisp required version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0c98d6226..b5c874919 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,7 @@ google = [ "google-cloud-speech~=2.32.0", "google-cloud-texttospeech~=2.26.0", " grok = [] groq = [ "groq~=0.23.0" ] gstreamer = [ "pygobject~=3.50.0" ] -krisp = [ "pipecat-ai-krisp~=0.3.0" ] +krisp = [ "pipecat-ai-krisp~=0.4.0" ] koala = [ "pvkoala~=2.0.3" ] langchain = [ "langchain~=0.3.20", "langchain-community~=0.3.20", "langchain-openai~=0.3.9" ] livekit = [ "livekit~=0.22.0", "livekit-api~=0.8.2", "tenacity~=9.0.0" ]