Deprecate expect_stripped_words option from LLMAssistantAggregatorParams, when used with the newer LLMAssistantAggregator, which now handles word spacing automatically.

This commit does not change how it works in the older `LLMAssistantContextAggregator`.
This commit is contained in:
Paul Kompfner
2025-10-30 15:48:15 -04:00
parent 5e00133e64
commit ac5734d0ed
18 changed files with 130 additions and 133 deletions

View File

@@ -33,13 +33,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
```python
context = LLMContext(messages, tools)
context_aggregator = LLMContextAggregatorPair(
context,
# This part is `OpenAIRealtimeLLMService`-specific.
# `expect_stripped_words=False` needed when OpenAI Realtime used with
# "audio" modality (the default).
assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False),
)
context_aggregator = LLMContextAggregatorPair(context)
```
(Note that even though `OpenAIRealtimeLLMService` now supports the universal
@@ -116,13 +110,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
```python
context = LLMContext(messages, tools)
context_aggregator = LLMContextAggregatorPair(
context,
# This part is `GeminiLiveLLMService`-specific.
# `expect_stripped_words=False` needed when Gemini Live used with AUDIO
# modality (the default).
assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False),
)
context_aggregator = LLMContextAggregatorPair(context)
```
(Note that even though `GeminiLiveLLMService` now supports the universal
@@ -202,6 +190,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Deprecated
- The `expect_stripped_words` parameter of `LLMAssistantAggregatorParams` is
ignored when used with the newer `LLMAssistantAggregator`, which now handles
word spacing automatically.
- `LLMService.request_image_frame()` is deprecated, push a
`UserImageRequestFrame` instead.

View File

@@ -187,12 +187,7 @@ Remember, your responses should be short. Just one or two sentences, usually. Re
tools,
)
context_aggregator = LLMContextAggregatorPair(
context,
# `expect_stripped_words=False` needed when OpenAI Realtime used with
# "audio" modality (the default)
assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False),
)
context_aggregator = LLMContextAggregatorPair(context)
pipeline = Pipeline(
[

View File

@@ -175,12 +175,7 @@ Remember, your responses should be short. Just one or two sentences, usually. Re
tools,
)
context_aggregator = LLMContextAggregatorPair(
context,
# `expect_stripped_words=False` needed when OpenAI Realtime used with
# "audio" modality (the default)
assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False),
)
context_aggregator = LLMContextAggregatorPair(context)
pipeline = Pipeline(
[

View File

@@ -92,12 +92,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
# },
],
)
context_aggregator = LLMContextAggregatorPair(
context,
# `expect_stripped_words=False` needed when Gemini Live used with AUDIO
# modality (the default)
assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False),
)
context_aggregator = LLMContextAggregatorPair(context)
transcript = TranscriptProcessor()

View File

@@ -144,12 +144,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
context = LLMContext(
[{"role": "user", "content": "Say hello."}],
)
context_aggregator = LLMContextAggregatorPair(
context,
# `expect_stripped_words=False` needed when Gemini Live used with AUDIO
# modality (the default)
assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False),
)
context_aggregator = LLMContextAggregatorPair(context)
pipeline = Pipeline(
[

View File

@@ -75,12 +75,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
},
],
)
context_aggregator = LLMContextAggregatorPair(
context,
# `expect_stripped_words=False` needed when Gemini Live used with AUDIO
# modality (the default)
assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False),
)
context_aggregator = LLMContextAggregatorPair(context)
pipeline = Pipeline(
[

View File

@@ -100,12 +100,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
}
],
)
context_aggregator = LLMContextAggregatorPair(
context,
# `expect_stripped_words=False` needed when Gemini Live used with AUDIO
# modality (the default)
assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False),
)
context_aggregator = LLMContextAggregatorPair(context)
pipeline = Pipeline(
[

View File

@@ -164,12 +164,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
)
# Create context aggregator
context_aggregator = LLMContextAggregatorPair(
context,
# `expect_stripped_words=False` needed when Gemini Live used with AUDIO
# modality (the default)
assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False),
)
context_aggregator = LLMContextAggregatorPair(context)
# Build the pipeline
pipeline = Pipeline(

View File

@@ -127,12 +127,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
# Set up conversation context and management
context = LLMContext(messages)
context_aggregator = LLMContextAggregatorPair(
context,
# `expect_stripped_words=False` needed when Gemini Live used with AUDIO
# modality (the default)
assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False),
)
context_aggregator = LLMContextAggregatorPair(context)
pipeline = Pipeline(
[

View File

@@ -140,12 +140,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation)
context = LLMContext([{"role": "user", "content": "Say hello."}])
context_aggregator = LLMContextAggregatorPair(
context,
# `expect_stripped_words=False` needed when Gemini Live used with AUDIO
# modality (the default)
assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False),
)
context_aggregator = LLMContextAggregatorPair(context)
pipeline = Pipeline(
[

View File

@@ -157,12 +157,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
context = LLMContext(
[{"role": "user", "content": "Say hello."}],
)
context_aggregator = LLMContextAggregatorPair(
context,
# `expect_stripped_words=False` needed when Gemini Live used with AUDIO
# modality (the default)
assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False),
)
context_aggregator = LLMContextAggregatorPair(context)
pipeline = Pipeline(
[

View File

@@ -111,12 +111,7 @@ async def run_bot(pipecat_transport):
]
context = LLMContext(messages)
context_aggregator = LLMContextAggregatorPair(
context,
# `expect_stripped_words=False` needed when Gemini Live used with AUDIO
# modality (the default)
assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False),
)
context_aggregator = LLMContextAggregatorPair(context)
# RTVI events for Pipecat client UI
rtvi = RTVIProcessor()

View File

@@ -89,7 +89,9 @@ class LLMAssistantAggregatorParams:
Parameters:
expect_stripped_words: Whether to expect and handle stripped words
in text frames by adding spaces between tokens.
in text frames by adding spaces between tokens. This parameter is
ignored when used with the newer LLMAssistantAggregator, which
handles word spacing automatically.
"""
expect_stripped_words: bool = True

View File

@@ -13,6 +13,7 @@ LLM processing, and text-to-speech components in conversational AI pipelines.
import asyncio
import json
import warnings
from abc import abstractmethod
from typing import Any, Dict, List, Literal, Optional, Set
@@ -65,6 +66,7 @@ from pipecat.processors.aggregators.llm_response import (
LLMUserAggregatorParams,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.utils.string import concatenate_aggregated_text
from pipecat.utils.time import time_now_iso8601
@@ -88,7 +90,7 @@ class LLMContextAggregator(FrameProcessor):
self._context = context
self._role = role
self._aggregation: str = ""
self._aggregation: List[str] = []
@property
def messages(self) -> List[LLMContextMessage]:
@@ -168,13 +170,21 @@ class LLMContextAggregator(FrameProcessor):
async def reset(self):
"""Reset the aggregation state."""
self._aggregation = ""
self._aggregation = []
@abstractmethod
async def push_aggregation(self):
"""Push the current aggregation downstream."""
pass
def aggregation_string(self) -> str:
"""Get the current aggregation as a string.
Returns:
The concatenated aggregation string.
"""
return concatenate_aggregated_text(self._aggregation)
class LLMUserAggregator(LLMContextAggregator):
"""User LLM aggregator that processes speech-to-text transcriptions.
@@ -212,8 +222,6 @@ class LLMUserAggregator(LLMContextAggregator):
self._turn_params: Optional[SmartTurnParams] = None
if "aggregation_timeout" in kwargs:
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
@@ -307,7 +315,7 @@ class LLMUserAggregator(LLMContextAggregator):
async def _process_aggregation(self):
"""Process the current aggregation and push it downstream."""
aggregation = self._aggregation
aggregation = self.aggregation_string()
await self.reset()
self._context.add_message({"role": self.role, "content": aggregation})
frame = LLMContextFrame(self._context)
@@ -355,7 +363,7 @@ class LLMUserAggregator(LLMContextAggregator):
"""
async def should_interrupt(strategy: BaseInterruptionStrategy):
await strategy.append_text(self._aggregation)
await strategy.append_text(self.aggregation_string())
return await strategy.should_interrupt()
return any([await should_interrupt(s) for s in self._interruption_strategies])
@@ -425,7 +433,7 @@ class LLMUserAggregator(LLMContextAggregator):
if not text.strip():
return
self._aggregation += f" {text}" if self._aggregation else text
self._aggregation.append(text)
# We just got a final result, so let's reset interim results.
self._seen_interim_results = False
# Reset aggregation timer.
@@ -550,23 +558,31 @@ class LLMAssistantAggregator(LLMContextAggregator):
Args:
context: The OpenAI LLM context for conversation storage.
params: Configuration parameters for aggregation behavior.
**kwargs: Additional arguments. Supports deprecated 'expect_stripped_words'.
**kwargs: Additional arguments.
"""
super().__init__(context=context, role="assistant", **kwargs)
self._params = params or LLMAssistantAggregatorParams()
if "expect_stripped_words" in kwargs:
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"Parameter 'expect_stripped_words' is deprecated, use 'params' instead.",
"Parameter 'expect_stripped_words' is deprecated. "
"LLMAssistantAggregator now handles word spacing automatically.",
DeprecationWarning,
)
self._params.expect_stripped_words = kwargs["expect_stripped_words"]
if params and not params.expect_stripped_words:
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"params.expect_stripped_words is deprecated. "
"LLMAssistantAggregator now handles word spacing automatically.",
DeprecationWarning,
)
self._started = 0
self._function_calls_in_progress: Dict[str, Optional[FunctionCallInProgressFrame]] = {}
self._context_updated_tasks: Set[asyncio.Task] = set()
@@ -629,7 +645,7 @@ class LLMAssistantAggregator(LLMContextAggregator):
if not self._aggregation:
return
aggregation = self._aggregation.strip()
aggregation = self.aggregation_string()
await self.reset()
if aggregation:
@@ -793,10 +809,11 @@ class LLMAssistantAggregator(LLMContextAggregator):
if not self._started:
return
if self._params.expect_stripped_words:
self._aggregation += f" {frame.text}" if self._aggregation else frame.text
else:
self._aggregation += frame.text
# Make sure we really have text (spaces count, too!)
if len(frame.text) == 0:
return
self._aggregation.append(frame.text)
def _context_updated_task_finished(self, task: asyncio.Task):
self._context_updated_tasks.discard(task)

View File

@@ -26,6 +26,7 @@ from pipecat.frames.frames import (
TTSTextFrame,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.utils.string import concatenate_aggregated_text
from pipecat.utils.time import time_now_iso8601
@@ -140,29 +141,7 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor):
Result: "Hello there how are you"
"""
if self._current_text_parts and self._aggregation_start_time:
# Check specifically for space characters, previously isspace() was used
# but that includes all whitespace characters (e.g. \n), not just spaces.
has_leading_spaces = any(
part and part[0] == " " for part in self._current_text_parts[1:]
)
has_trailing_spaces = any(
part and part[-1] == " " for part in self._current_text_parts[:-1]
)
# If there are embedded spaces in the fragments, use direct concatenation
contains_spacing_between_fragments = has_leading_spaces or has_trailing_spaces
# Apply corresponding joining method
if contains_spacing_between_fragments:
# Fragments already have spacing - just concatenate
content = "".join(self._current_text_parts)
else:
# Word-by-word fragments - join with spaces
content = " ".join(self._current_text_parts)
# Clean up any excessive whitespace
content = content.strip()
content = concatenate_aggregated_text(self._current_text_parts)
if content:
logger.trace(f"Emitting aggregated assistant message: {content}")
message = TranscriptionMessage(

View File

@@ -18,7 +18,7 @@ Dependencies:
"""
import re
from typing import FrozenSet, Optional, Sequence, Tuple
from typing import FrozenSet, List, Optional, Sequence, Tuple
import nltk
from loguru import logger
@@ -196,3 +196,40 @@ def parse_start_end_tags(
return (None, len(text))
return (None, current_tag_index)
def concatenate_aggregated_text(text_parts: List[str]) -> str:
"""Concatenate a list of text parts into a single string.
This function joins the provided list of text parts into a single string,
taking into account whether or not the parts already contain spacing.
This function is useful for aggregating text segments received from LLMs or
transcription services.
Args:
text_parts: A list of strings representing parts of text to concatenate.
Returns:
A single concatenated string.
"""
# Check specifically for space characters, previously isspace() was used
# but that includes all whitespace characters (e.g. \n), not just spaces.
has_leading_spaces = any(part and part[0] == " " for part in text_parts[1:])
has_trailing_spaces = any(part and part[-1] == " " for part in text_parts[:-1])
# If there are embedded spaces in the fragments, use direct concatenation
contains_spacing_between_fragments = has_leading_spaces or has_trailing_spaces
# Apply corresponding joining method
if contains_spacing_between_fragments:
# Fragments already have spacing - just concatenate
result = "".join(text_parts)
else:
# Word-by-word fragments - join with spaces
result = " ".join(text_parts)
# Clean up any excessive whitespace
result = result.strip()
return result

View File

@@ -6,7 +6,7 @@
import json
import unittest
from typing import Any
from typing import Any, Optional
from pipecat.audio.interruptions.min_words_interruption_strategy import MinWordsInterruptionStrategy
from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams
@@ -22,6 +22,8 @@ from pipecat.frames.frames import (
InterimTranscriptionFrame,
InterruptionFrame,
InterruptionTaskFrame,
LLMContextAssistantTimestampFrame,
LLMContextFrame,
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
OpenAILLMContextAssistantTimestampFrame,
@@ -38,6 +40,7 @@ from pipecat.processors.aggregators.llm_response import (
LLMUserAggregatorParams,
LLMUserContextAggregator,
)
from pipecat.processors.aggregators.llm_response_universal import LLMAssistantAggregator
from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContext,
OpenAILLMContextFrame,
@@ -586,11 +589,16 @@ class BaseTestUserContextAggregator:
assert context_processor.context_received
class BaseTestAssistantContextAggreagator:
class BaseTestAssistantContextAggregator:
CONTEXT_CLASS = None # To be set in subclasses
AGGREGATOR_CLASS = None # To be set in subclasses
EXPECTED_CONTEXT_FRAMES = None # To be set in subclasses
def create_assistant_aggregator_params(
self, **kwargs
) -> Optional[LLMAssistantAggregatorParams]:
return LLMAssistantAggregatorParams(**kwargs)
def check_message_content(self, context: OpenAILLMContext, index: int, content: str):
assert context.messages[index]["content"] == content
@@ -641,7 +649,7 @@ class BaseTestAssistantContextAggreagator:
context = self.CONTEXT_CLASS()
aggregator = self.AGGREGATOR_CLASS(
context, params=LLMAssistantAggregatorParams(expect_stripped_words=False)
context, params=self.create_assistant_aggregator_params(expect_stripped_words=False)
)
frames_to_send = [
LLMFullResponseStartFrame(),
@@ -687,7 +695,7 @@ class BaseTestAssistantContextAggreagator:
context = self.CONTEXT_CLASS()
aggregator = self.AGGREGATOR_CLASS(
context, params=LLMAssistantAggregatorParams(expect_stripped_words=False)
context, params=self.create_assistant_aggregator_params(expect_stripped_words=False)
)
frames_to_send = [
LLMFullResponseStartFrame(),
@@ -714,7 +722,7 @@ class BaseTestAssistantContextAggreagator:
context = self.CONTEXT_CLASS()
aggregator = self.AGGREGATOR_CLASS(
context, params=LLMAssistantAggregatorParams(expect_stripped_words=False)
context, params=self.create_assistant_aggregator_params(expect_stripped_words=False)
)
frames_to_send = [
LLMFullResponseStartFrame(),
@@ -838,7 +846,7 @@ class TestAnthropicUserContextAggregator(
class TestAnthropicAssistantContextAggregator(
BaseTestAssistantContextAggreagator, unittest.IsolatedAsyncioTestCase
BaseTestAssistantContextAggregator, unittest.IsolatedAsyncioTestCase
):
CONTEXT_CLASS = AnthropicLLMContext
AGGREGATOR_CLASS = AnthropicAssistantContextAggregator
@@ -873,7 +881,7 @@ class TestAWSBedrockUserContextAggregator(
class TestAWSBedrockAssistantContextAggregator(
BaseTestAssistantContextAggreagator, unittest.IsolatedAsyncioTestCase
BaseTestAssistantContextAggregator, unittest.IsolatedAsyncioTestCase
):
CONTEXT_CLASS = AWSBedrockLLMContext
AGGREGATOR_CLASS = AWSBedrockAssistantContextAggregator
@@ -914,7 +922,7 @@ class TestGoogleUserContextAggregator(
class TestGoogleAssistantContextAggregator(
BaseTestAssistantContextAggreagator, unittest.IsolatedAsyncioTestCase
BaseTestAssistantContextAggregator, unittest.IsolatedAsyncioTestCase
):
CONTEXT_CLASS = GoogleLLMContext
AGGREGATOR_CLASS = GoogleAssistantContextAggregator
@@ -948,8 +956,27 @@ class TestOpenAIUserContextAggregator(
class TestOpenAIAssistantContextAggregator(
BaseTestAssistantContextAggreagator, unittest.IsolatedAsyncioTestCase
BaseTestAssistantContextAggregator, unittest.IsolatedAsyncioTestCase
):
CONTEXT_CLASS = OpenAILLMContext
AGGREGATOR_CLASS = OpenAIAssistantContextAggregator
EXPECTED_CONTEXT_FRAMES = [OpenAILLMContextFrame, OpenAILLMContextAssistantTimestampFrame]
#
# Universal
#
class TestLLMAssistantAggregator(
BaseTestAssistantContextAggregator, unittest.IsolatedAsyncioTestCase
):
CONTEXT_CLASS = OpenAILLMContext
AGGREGATOR_CLASS = LLMAssistantAggregator
EXPECTED_CONTEXT_FRAMES = [LLMContextFrame, LLMContextAssistantTimestampFrame]
# Override to remove 'expect_stripped_words' parameter, which is deprecated
# for LLMAssistantAggregator
def create_assistant_aggregator_params(
self, **kwargs
) -> Optional[LLMAssistantAggregatorParams]:
kwargs.pop("expect_stripped_words", None)
return LLMAssistantAggregatorParams(**kwargs) if kwargs else None

View File

@@ -65,9 +65,7 @@ class TestLangchain(unittest.IsolatedAsyncioTestCase):
self.mock_proc = self.MockProcessor("token_collector")
context = LLMContext()
context_aggregator = LLMContextAggregatorPair(
context, assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False)
)
context_aggregator = LLMContextAggregatorPair(context)
pipeline = Pipeline(
[context_aggregator.user(), proc, self.mock_proc, context_aggregator.assistant()]