Always apply a timeout to summarization LLM calls

Even when summarization_timeout is explicitly set to None, use a
DEFAULT_SUMMARIZATION_TIMEOUT (120s) fallback so the LLM call can
never hang indefinitely. Applied in both LLMService and the dedicated
LLM path in LLMContextSummarizer.
This commit is contained in:
Mark Backman
2026-02-27 11:57:22 -05:00
parent 82c249608f
commit f74af9b9c7
4 changed files with 21 additions and 21 deletions

View File

@@ -2020,7 +2020,7 @@ class LLMContextSummaryRequestFrame(ControlFrame):
summarization_prompt: System prompt instructing the LLM how to generate summarization_prompt: System prompt instructing the LLM how to generate
the summary. the summary.
summarization_timeout: Maximum time in seconds for the LLM to generate a summarization_timeout: Maximum time in seconds for the LLM to generate a
summary. None means no timeout. summary. When None, a default timeout of 120s is applied.
""" """
request_id: str request_id: str

View File

@@ -24,6 +24,7 @@ from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMe
from pipecat.utils.asyncio.task_manager import BaseTaskManager from pipecat.utils.asyncio.task_manager import BaseTaskManager
from pipecat.utils.base_object import BaseObject from pipecat.utils.base_object import BaseObject
from pipecat.utils.context.llm_context_summarization import ( from pipecat.utils.context.llm_context_summarization import (
DEFAULT_SUMMARIZATION_TIMEOUT,
LLMContextSummarizationConfig, LLMContextSummarizationConfig,
LLMContextSummarizationUtil, LLMContextSummarizationUtil,
) )
@@ -278,21 +279,20 @@ class LLMContextSummarizer(BaseObject):
llm: The dedicated LLM service to use for summarization. llm: The dedicated LLM service to use for summarization.
frame: The summarization request frame. frame: The summarization request frame.
""" """
timeout = frame.summarization_timeout or DEFAULT_SUMMARIZATION_TIMEOUT
try: try:
if frame.summarization_timeout: summary, last_index = await asyncio.wait_for(
summary, last_index = await asyncio.wait_for( llm._generate_summary(frame),
llm._generate_summary(frame), timeout=timeout,
timeout=frame.summarization_timeout, )
)
else:
summary, last_index = await llm._generate_summary(frame)
result_frame = LLMContextSummaryResultFrame( result_frame = LLMContextSummaryResultFrame(
request_id=frame.request_id, request_id=frame.request_id,
summary=summary, summary=summary,
last_summarized_index=last_index, last_summarized_index=last_index,
) )
except asyncio.TimeoutError: except asyncio.TimeoutError:
error = f"Context summarization timed out after {frame.summarization_timeout}s" error = f"Context summarization timed out after {timeout}s"
logger.error(f"{self}: {error}") logger.error(f"{self}: {error}")
result_frame = LLMContextSummaryResultFrame( result_frame = LLMContextSummaryResultFrame(
request_id=frame.request_id, request_id=frame.request_id,

View File

@@ -62,6 +62,7 @@ from pipecat.services.ai_service import AIService
from pipecat.services.settings import LLMSettings from pipecat.services.settings import LLMSettings
from pipecat.turns.user_turn_completion_mixin import UserTurnCompletionLLMServiceMixin from pipecat.turns.user_turn_completion_mixin import UserTurnCompletionLLMServiceMixin
from pipecat.utils.context.llm_context_summarization import ( from pipecat.utils.context.llm_context_summarization import (
DEFAULT_SUMMARIZATION_TIMEOUT,
LLMContextSummarizationUtil, LLMContextSummarizationUtil,
) )
@@ -436,18 +437,15 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
last_index = -1 last_index = -1
error = None error = None
timeout = frame.summarization_timeout or DEFAULT_SUMMARIZATION_TIMEOUT
try: try:
if frame.summarization_timeout: summary, last_index = await asyncio.wait_for(
summary, last_index = await asyncio.wait_for( self._generate_summary(frame),
self._generate_summary(frame), timeout=timeout,
timeout=frame.summarization_timeout,
)
else:
summary, last_index = await self._generate_summary(frame)
except asyncio.TimeoutError:
await self.push_error(
error_msg=f"Context summarization timed out after {frame.summarization_timeout}s"
) )
except asyncio.TimeoutError:
await self.push_error(error_msg=f"Context summarization timed out after {timeout}s")
except Exception as e: except Exception as e:
error = f"Error generating context summary: {e}" error = f"Error generating context summary: {e}"
await self.push_error(error, exception=e) await self.push_error(error, exception=e)

View File

@@ -20,6 +20,9 @@ from loguru import logger
from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage
# Fallback timeout (seconds) used when summarization_timeout is None.
DEFAULT_SUMMARIZATION_TIMEOUT = 120.0
# Token estimation constants # Token estimation constants
CHARS_PER_TOKEN = 4 # Industry-standard heuristic: 1 token ≈ 4 characters CHARS_PER_TOKEN = 4 # Industry-standard heuristic: 1 token ≈ 4 characters
TOKEN_OVERHEAD_PER_MESSAGE = 10 # Estimated structural overhead per message TOKEN_OVERHEAD_PER_MESSAGE = 10 # Estimated structural overhead per message
@@ -89,7 +92,6 @@ class LLMContextSummarizationConfig:
summarization_timeout: Maximum time in seconds to wait for the LLM to summarization_timeout: Maximum time in seconds to wait for the LLM to
generate a summary. If the call exceeds this timeout, summarization generate a summary. If the call exceeds this timeout, summarization
is aborted with an error and future summarizations are unblocked. is aborted with an error and future summarizations are unblocked.
Set to None to disable the timeout.
""" """
max_context_tokens: int = 8000 max_context_tokens: int = 8000
@@ -99,7 +101,7 @@ class LLMContextSummarizationConfig:
summarization_prompt: Optional[str] = None summarization_prompt: Optional[str] = None
summary_message_template: str = "Conversation summary: {summary}" summary_message_template: str = "Conversation summary: {summary}"
llm: Optional["LLMService"] = None llm: Optional["LLMService"] = None
summarization_timeout: Optional[float] = 120.0 summarization_timeout: float = DEFAULT_SUMMARIZATION_TIMEOUT
def __post_init__(self): def __post_init__(self):
"""Validate configuration parameters.""" """Validate configuration parameters."""