diff --git a/CHANGELOG.md b/CHANGELOG.md index f208bf1d9..6ff37d47c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ```python context = LLMContext(messages, tools) - context_aggregator = LLMContextAggregatorPair(context) + context_aggregator = LLMContextAggregatorPair( + context, + # This part is `OpenAIRealtimeLLMService`-specific. + # `expect_stripped_words=False` needed when OpenAI Realtime used with + # "audio" modality (the default). + assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False), + ) ``` (Note that even though `OpenAIRealtimeLLMService` now supports the universal diff --git a/examples/foundational/19-openai-realtime.py b/examples/foundational/19-openai-realtime.py index 5f215a07b..3451c9da8 100644 --- a/examples/foundational/19-openai-realtime.py +++ b/examples/foundational/19-openai-realtime.py @@ -21,6 +21,7 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response import LLMAssistantAggregatorParams from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments @@ -186,7 +187,12 @@ Remember, your responses should be short. Just one or two sentences, usually. Re tools, ) - context_aggregator = LLMContextAggregatorPair(context) + context_aggregator = LLMContextAggregatorPair( + context, + # `expect_stripped_words=False` needed when OpenAI Realtime used with + # "audio" modality (the default) + assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False), + ) pipeline = Pipeline( [ diff --git a/examples/foundational/19a-azure-realtime.py b/examples/foundational/19a-azure-realtime.py index 7b07985be..7d9cf1b4b 100644 --- a/examples/foundational/19a-azure-realtime.py +++ b/examples/foundational/19a-azure-realtime.py @@ -19,6 +19,7 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response import LLMAssistantAggregatorParams from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -174,7 +175,12 @@ Remember, your responses should be short. Just one or two sentences, usually. Re tools, ) - context_aggregator = LLMContextAggregatorPair(context) + context_aggregator = LLMContextAggregatorPair( + context, + # `expect_stripped_words=False` needed when OpenAI Realtime used with + # "audio" modality (the default) + assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False), + ) pipeline = Pipeline( [ diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index c46f3435d..f9e8b1a3f 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -895,6 +895,7 @@ class OpenAIRealtimeLLMService(LLMService): ) context = LLMContext.from_openai_context(context) + assistant_params.expect_stripped_words = False return LLMContextAggregatorPair( context, user_params=user_params, assistant_params=assistant_params )