Always apply a timeout to summarization LLM calls

Even when summarization_timeout is explicitly set to None, use a
DEFAULT_SUMMARIZATION_TIMEOUT (120s) fallback so the LLM call can
never hang indefinitely. Applied in both LLMService and the dedicated
LLM path in LLMContextSummarizer.
This commit is contained in:
Mark Backman
2026-02-27 11:57:22 -05:00
parent 82c249608f
commit f74af9b9c7
4 changed files with 21 additions and 21 deletions

View File

@@ -2020,7 +2020,7 @@ class LLMContextSummaryRequestFrame(ControlFrame):
summarization_prompt: System prompt instructing the LLM how to generate
the summary.
summarization_timeout: Maximum time in seconds for the LLM to generate a
summary. None means no timeout.
summary. When None, a default timeout of 120s is applied.
"""
request_id: str

View File

@@ -24,6 +24,7 @@ from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMe
from pipecat.utils.asyncio.task_manager import BaseTaskManager
from pipecat.utils.base_object import BaseObject
from pipecat.utils.context.llm_context_summarization import (
DEFAULT_SUMMARIZATION_TIMEOUT,
LLMContextSummarizationConfig,
LLMContextSummarizationUtil,
)
@@ -278,21 +279,20 @@ class LLMContextSummarizer(BaseObject):
llm: The dedicated LLM service to use for summarization.
frame: The summarization request frame.
"""
timeout = frame.summarization_timeout or DEFAULT_SUMMARIZATION_TIMEOUT
try:
if frame.summarization_timeout:
summary, last_index = await asyncio.wait_for(
llm._generate_summary(frame),
timeout=frame.summarization_timeout,
)
else:
summary, last_index = await llm._generate_summary(frame)
summary, last_index = await asyncio.wait_for(
llm._generate_summary(frame),
timeout=timeout,
)
result_frame = LLMContextSummaryResultFrame(
request_id=frame.request_id,
summary=summary,
last_summarized_index=last_index,
)
except asyncio.TimeoutError:
error = f"Context summarization timed out after {frame.summarization_timeout}s"
error = f"Context summarization timed out after {timeout}s"
logger.error(f"{self}: {error}")
result_frame = LLMContextSummaryResultFrame(
request_id=frame.request_id,

View File

@@ -62,6 +62,7 @@ from pipecat.services.ai_service import AIService
from pipecat.services.settings import LLMSettings
from pipecat.turns.user_turn_completion_mixin import UserTurnCompletionLLMServiceMixin
from pipecat.utils.context.llm_context_summarization import (
DEFAULT_SUMMARIZATION_TIMEOUT,
LLMContextSummarizationUtil,
)
@@ -436,18 +437,15 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
last_index = -1
error = None
timeout = frame.summarization_timeout or DEFAULT_SUMMARIZATION_TIMEOUT
try:
if frame.summarization_timeout:
summary, last_index = await asyncio.wait_for(
self._generate_summary(frame),
timeout=frame.summarization_timeout,
)
else:
summary, last_index = await self._generate_summary(frame)
except asyncio.TimeoutError:
await self.push_error(
error_msg=f"Context summarization timed out after {frame.summarization_timeout}s"
summary, last_index = await asyncio.wait_for(
self._generate_summary(frame),
timeout=timeout,
)
except asyncio.TimeoutError:
await self.push_error(error_msg=f"Context summarization timed out after {timeout}s")
except Exception as e:
error = f"Error generating context summary: {e}"
await self.push_error(error, exception=e)

View File

@@ -20,6 +20,9 @@ from loguru import logger
from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage
# Fallback timeout (seconds) used when summarization_timeout is None.
DEFAULT_SUMMARIZATION_TIMEOUT = 120.0
# Token estimation constants
CHARS_PER_TOKEN = 4 # Industry-standard heuristic: 1 token ≈ 4 characters
TOKEN_OVERHEAD_PER_MESSAGE = 10 # Estimated structural overhead per message
@@ -89,7 +92,6 @@ class LLMContextSummarizationConfig:
summarization_timeout: Maximum time in seconds to wait for the LLM to
generate a summary. If the call exceeds this timeout, summarization
is aborted with an error and future summarizations are unblocked.
Set to None to disable the timeout.
"""
max_context_tokens: int = 8000
@@ -99,7 +101,7 @@ class LLMContextSummarizationConfig:
summarization_prompt: Optional[str] = None
summary_message_template: str = "Conversation summary: {summary}"
llm: Optional["LLMService"] = None
summarization_timeout: Optional[float] = 120.0
summarization_timeout: float = DEFAULT_SUMMARIZATION_TIMEOUT
def __post_init__(self):
"""Validate configuration parameters."""