diff --git a/changelog/4219.added.md b/changelog/4219.added.md new file mode 100644 index 000000000..ddf4bc0fa --- /dev/null +++ b/changelog/4219.added.md @@ -0,0 +1 @@ +- Added `enable_prompt_caching` setting to `AWSBedrockLLMService` for Bedrock ConverseStream prompt caching. diff --git a/src/pipecat/services/aws/llm.py b/src/pipecat/services/aws/llm.py index 722072592..52c95070d 100644 --- a/src/pipecat/services/aws/llm.py +++ b/src/pipecat/services/aws/llm.py @@ -36,6 +36,7 @@ from pipecat.frames.frames import ( FunctionCallInProgressFrame, FunctionCallResultFrame, LLMContextFrame, + LLMEnablePromptCachingFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, UserImageRawFrame, @@ -66,11 +67,16 @@ class AWSBedrockLLMSettings(LLMSettings): Parameters: stop_sequences: List of strings that stop generation. latency: Performance mode - "standard" or "optimized". + enable_prompt_caching: Whether to enable prompt caching by adding cachePoint + markers to system prompts and tool definitions. Can reduce TTFT by up to + 85% for multi-turn conversations. See: + https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html additional_model_request_fields: Additional model-specific parameters. """ stop_sequences: List[str] | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) latency: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + enable_prompt_caching: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN) additional_model_request_fields: Dict[str, Any] | _NotGiven = field( default_factory=lambda: NOT_GIVEN ) @@ -174,6 +180,7 @@ class AWSBedrockLLMService(LLMService): user_turn_completion_config=None, stop_sequences=None, latency=None, + enable_prompt_caching=False, additional_model_request_fields={}, ) @@ -455,6 +462,24 @@ class AWSBedrockLLMService(LLMService): if self._settings.latency in ["standard", "optimized"]: request_params["performanceConfig"] = {"latency": self._settings.latency} + # Add cache checkpoints to system prompts and tool definitions. + # This enables prompt caching for providers that support it (e.g. + # Anthropic Claude on Bedrock), reducing TTFT by up to 85% on + # multi-turn conversations where the system prompt stays constant. + if self._settings.enable_prompt_caching: + if "system" in request_params and request_params["system"]: + system_list = request_params["system"] + if not any("cachePoint" in item for item in system_list): + system_list.append({"cachePoint": {"type": "default"}}) + if ( + "toolConfig" in request_params + and "tools" in request_params["toolConfig"] + and request_params["toolConfig"]["tools"] + ): + tools_list = request_params["toolConfig"]["tools"] + if not any("cachePoint" in t for t in tools_list): + tools_list.append({"cachePoint": {"type": "default"}}) + # Log request params with messages redacted for logging adapter = self.get_llm_adapter() messages_for_logging = adapter.get_messages_for_logging(context) @@ -566,6 +591,9 @@ class AWSBedrockLLMService(LLMService): if isinstance(frame, LLMContextFrame): await self._process_context(frame.context) + elif isinstance(frame, LLMEnablePromptCachingFrame): + logger.debug(f"Setting enable prompt caching to: [{frame.enable}]") + self._settings.enable_prompt_caching = frame.enable else: await self.push_frame(frame, direction)