diff --git a/CHANGELOG.md b/CHANGELOG.md index 855ef02dc..c3f8fc3fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,10 +34,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- `OpenAIRealtimeBetaLLMService` and `GeminiMultimodalLiveLLMService` no longer - push `LLMTextFrame`. Instead, they both push only `TTSTextFrame`, which is - used to aggregate the assistant context and generate a transcript. - - Function calls now receive a single parameter `FunctionCallParams` instead of `(function_name, tool_call_id, args, llm, context, result_callback)` which is now deprecated. diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index b5b650ef5..01f55407e 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -30,6 +30,7 @@ from pipecat.frames.frames import ( LLMFullResponseStartFrame, LLMMessagesAppendFrame, LLMSetToolsFrame, + LLMTextFrame, LLMUpdateSettingsFrame, StartFrame, StartInterruptionFrame, @@ -222,6 +223,14 @@ class GeminiMultimodalLiveUserContextAggregator(OpenAIUserContextAggregator): class GeminiMultimodalLiveAssistantContextAggregator(OpenAIAssistantContextAggregator): + # The LLMAssistantContextAggregator uses TextFrames to aggregate the LLM output, + # but the GeminiMultimodalLiveAssistantContextAggregator pushes LLMTextFrames and TTSTextFrames. We + # need to override this proces_frame for LLMTextFrame, so that only the TTSTextFrames + # are process. This ensures that the context gets only one set of messages. + async def process_frame(self, frame: Frame, direction: FrameDirection): + if not isinstance(frame, LLMTextFrame): + await super().process_frame(frame, direction) + async def handle_user_image_frame(self, frame: UserImageRawFrame): # We don't want to store any images in the context. Revisit this later # when the API evolves. @@ -365,7 +374,7 @@ class GeminiMultimodalLiveLLMService(LLMService): "vad": params.vad, "context_window_compression": params.context_window_compression.model_dump() if params.context_window_compression - else None, + else {}, "extra": params.extra if isinstance(params.extra, dict) else {}, } @@ -891,6 +900,7 @@ class GeminiMultimodalLiveLLMService(LLMService): if not text: return + await self.push_frame(LLMTextFrame(text=text)) await self.push_frame(TTSTextFrame(text=text)) def create_context_aggregator( diff --git a/src/pipecat/services/openai_realtime_beta/context.py b/src/pipecat/services/openai_realtime_beta/context.py index 80cffbef2..d9e105098 100644 --- a/src/pipecat/services/openai_realtime_beta/context.py +++ b/src/pipecat/services/openai_realtime_beta/context.py @@ -14,6 +14,7 @@ from pipecat.frames.frames import ( FunctionCallResultFrame, LLMMessagesUpdateFrame, LLMSetToolsFrame, + LLMTextFrame, ) from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection @@ -170,6 +171,14 @@ class OpenAIRealtimeUserContextAggregator(OpenAIUserContextAggregator): class OpenAIRealtimeAssistantContextAggregator(OpenAIAssistantContextAggregator): + # The LLMAssistantContextAggregator uses TextFrames to aggregate the LLM output, + # but the OpenAIRealtimeLLMService pushes LLMTextFrames and TTSTextFrames. We + # need to override this proces_frame for LLMTextFrame, so that only the TTSTextFrames + # are process. This ensures that the context gets only one set of messages. + async def process_frame(self, frame: Frame, direction: FrameDirection): + if not isinstance(frame, LLMTextFrame): + await super().process_frame(frame, direction) + async def handle_function_call_result(self, frame: FunctionCallResultFrame): await super().handle_function_call_result(frame) diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index a4b339362..94d848b77 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -24,6 +24,7 @@ from pipecat.frames.frames import ( LLMFullResponseStartFrame, LLMMessagesAppendFrame, LLMSetToolsFrame, + LLMTextFrame, LLMUpdateSettingsFrame, StartFrame, StartInterruptionFrame, @@ -524,6 +525,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): async def _handle_evt_audio_transcript_delta(self, evt): if evt.delta: + await self.push_frame(LLMTextFrame(evt.delta)) await self.push_frame(TTSTextFrame(evt.delta)) async def _handle_evt_speech_started(self, evt):