Always apply a timeout to summarization LLM calls
Even when summarization_timeout is explicitly set to None, use a DEFAULT_SUMMARIZATION_TIMEOUT (120s) fallback so the LLM call can never hang indefinitely. Applied in both LLMService and the dedicated LLM path in LLMContextSummarizer.
This commit is contained in:
@@ -2020,7 +2020,7 @@ class LLMContextSummaryRequestFrame(ControlFrame):
|
||||
summarization_prompt: System prompt instructing the LLM how to generate
|
||||
the summary.
|
||||
summarization_timeout: Maximum time in seconds for the LLM to generate a
|
||||
summary. None means no timeout.
|
||||
summary. When None, a default timeout of 120s is applied.
|
||||
"""
|
||||
|
||||
request_id: str
|
||||
|
||||
@@ -24,6 +24,7 @@ from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMe
|
||||
from pipecat.utils.asyncio.task_manager import BaseTaskManager
|
||||
from pipecat.utils.base_object import BaseObject
|
||||
from pipecat.utils.context.llm_context_summarization import (
|
||||
DEFAULT_SUMMARIZATION_TIMEOUT,
|
||||
LLMContextSummarizationConfig,
|
||||
LLMContextSummarizationUtil,
|
||||
)
|
||||
@@ -278,21 +279,20 @@ class LLMContextSummarizer(BaseObject):
|
||||
llm: The dedicated LLM service to use for summarization.
|
||||
frame: The summarization request frame.
|
||||
"""
|
||||
timeout = frame.summarization_timeout or DEFAULT_SUMMARIZATION_TIMEOUT
|
||||
|
||||
try:
|
||||
if frame.summarization_timeout:
|
||||
summary, last_index = await asyncio.wait_for(
|
||||
llm._generate_summary(frame),
|
||||
timeout=frame.summarization_timeout,
|
||||
)
|
||||
else:
|
||||
summary, last_index = await llm._generate_summary(frame)
|
||||
summary, last_index = await asyncio.wait_for(
|
||||
llm._generate_summary(frame),
|
||||
timeout=timeout,
|
||||
)
|
||||
result_frame = LLMContextSummaryResultFrame(
|
||||
request_id=frame.request_id,
|
||||
summary=summary,
|
||||
last_summarized_index=last_index,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
error = f"Context summarization timed out after {frame.summarization_timeout}s"
|
||||
error = f"Context summarization timed out after {timeout}s"
|
||||
logger.error(f"{self}: {error}")
|
||||
result_frame = LLMContextSummaryResultFrame(
|
||||
request_id=frame.request_id,
|
||||
|
||||
@@ -62,6 +62,7 @@ from pipecat.services.ai_service import AIService
|
||||
from pipecat.services.settings import LLMSettings
|
||||
from pipecat.turns.user_turn_completion_mixin import UserTurnCompletionLLMServiceMixin
|
||||
from pipecat.utils.context.llm_context_summarization import (
|
||||
DEFAULT_SUMMARIZATION_TIMEOUT,
|
||||
LLMContextSummarizationUtil,
|
||||
)
|
||||
|
||||
@@ -436,18 +437,15 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
|
||||
last_index = -1
|
||||
error = None
|
||||
|
||||
timeout = frame.summarization_timeout or DEFAULT_SUMMARIZATION_TIMEOUT
|
||||
|
||||
try:
|
||||
if frame.summarization_timeout:
|
||||
summary, last_index = await asyncio.wait_for(
|
||||
self._generate_summary(frame),
|
||||
timeout=frame.summarization_timeout,
|
||||
)
|
||||
else:
|
||||
summary, last_index = await self._generate_summary(frame)
|
||||
except asyncio.TimeoutError:
|
||||
await self.push_error(
|
||||
error_msg=f"Context summarization timed out after {frame.summarization_timeout}s"
|
||||
summary, last_index = await asyncio.wait_for(
|
||||
self._generate_summary(frame),
|
||||
timeout=timeout,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
await self.push_error(error_msg=f"Context summarization timed out after {timeout}s")
|
||||
except Exception as e:
|
||||
error = f"Error generating context summary: {e}"
|
||||
await self.push_error(error, exception=e)
|
||||
|
||||
@@ -20,6 +20,9 @@ from loguru import logger
|
||||
|
||||
from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage
|
||||
|
||||
# Fallback timeout (seconds) used when summarization_timeout is None.
|
||||
DEFAULT_SUMMARIZATION_TIMEOUT = 120.0
|
||||
|
||||
# Token estimation constants
|
||||
CHARS_PER_TOKEN = 4 # Industry-standard heuristic: 1 token ≈ 4 characters
|
||||
TOKEN_OVERHEAD_PER_MESSAGE = 10 # Estimated structural overhead per message
|
||||
@@ -89,7 +92,6 @@ class LLMContextSummarizationConfig:
|
||||
summarization_timeout: Maximum time in seconds to wait for the LLM to
|
||||
generate a summary. If the call exceeds this timeout, summarization
|
||||
is aborted with an error and future summarizations are unblocked.
|
||||
Set to None to disable the timeout.
|
||||
"""
|
||||
|
||||
max_context_tokens: int = 8000
|
||||
@@ -99,7 +101,7 @@ class LLMContextSummarizationConfig:
|
||||
summarization_prompt: Optional[str] = None
|
||||
summary_message_template: str = "Conversation summary: {summary}"
|
||||
llm: Optional["LLMService"] = None
|
||||
summarization_timeout: Optional[float] = 120.0
|
||||
summarization_timeout: float = DEFAULT_SUMMARIZATION_TIMEOUT
|
||||
|
||||
def __post_init__(self):
|
||||
"""Validate configuration parameters."""
|
||||
|
||||
Reference in New Issue
Block a user