diff --git a/CHANGELOG.md b/CHANGELOG.md index 14b779f47..8ae588300 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added support for switching between audio+text to text-only modes within the + same pipeline. This is done by pushing + `LLMConfigureOutputFrame(skip_tts=True)` to enter text-only mode, and + disabling it to return to audio+text. The LLM will still generate tokens and + add them to the context, but they will not be sent to TTS. + - Added `skip_tts` field to `TextFrame`. This lets a text frame bypass TTS while still being included in the LLM context. Useful for cases like structured text that isn’t meant to be spoken but should still contribute to context. diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 18ab13bac..5052c0df2 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -607,6 +607,21 @@ class LLMEnablePromptCachingFrame(DataFrame): enable: bool +@dataclass +class LLMConfigureOutputFrame(DataFrame): + """Frame to configure LLM output. + + This frame is used to configure how the LLM produces output. For example, it + can tell the LLM to generate tokens that should be added to the context but + not spoken by the TTS service (if one is present in the pipeline). + + Parameters: + skip_tts: Whether LLM tokens should skip the TTS service (if any). + """ + + skip_tts: bool + + @dataclass class TTSSpeakFrame(DataFrame): """Frame containing text that should be spoken by TTS. diff --git a/src/pipecat/processors/aggregators/dtmf_aggregator.py b/src/pipecat/processors/aggregators/dtmf_aggregator.py index 24ef2a1e1..38e1296f6 100644 --- a/src/pipecat/processors/aggregators/dtmf_aggregator.py +++ b/src/pipecat/processors/aggregators/dtmf_aggregator.py @@ -103,7 +103,7 @@ class DTMFAggregator(FrameProcessor): digit_value = frame.button.value self._aggregation += digit_value - # For first digit, schedule interruption in separate task + # For first digit, schedule interruption. if is_first_digit: await self.push_frame(BotInterruptionFrame(), FrameDirection.UPSTREAM) diff --git a/src/pipecat/services/llm_service.py b/src/pipecat/services/llm_service.py index 3152a0083..63f7659b3 100644 --- a/src/pipecat/services/llm_service.py +++ b/src/pipecat/services/llm_service.py @@ -37,6 +37,8 @@ from pipecat.frames.frames import ( FunctionCallResultFrame, FunctionCallResultProperties, FunctionCallsStartedFrame, + LLMConfigureOutputFrame, + LLMTextFrame, StartFrame, StartInterruptionFrame, UserImageRequestFrame, @@ -179,6 +181,7 @@ class LLMService(AIService): self._function_call_tasks: Dict[asyncio.Task, FunctionCallRunnerItem] = {} self._sequential_runner_task: Optional[asyncio.Task] = None self._tracing_enabled: bool = False + self._skip_tts: bool = False self._register_event_handler("on_function_calls_started") self._register_event_handler("on_completion_timeout") @@ -272,6 +275,20 @@ class LLMService(AIService): if isinstance(frame, StartInterruptionFrame): await self._handle_interruptions(frame) + elif isinstance(frame, LLMConfigureOutputFrame): + self._skip_tts = frame.skip_tts + + async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): + """Pushes a frame. + + Args: + frame: The frame to push. + direction: The direction of frame pushing. + """ + if isinstance(frame, LLMTextFrame): + frame.skip_tts = self._skip_tts + + await super().push_frame(frame, direction) async def _handle_interruptions(self, _: StartInterruptionFrame): for function_name, entry in self._functions.items():