Always apply a timeout to summarization LLM calls
Even when summarization_timeout is explicitly set to None, use a DEFAULT_SUMMARIZATION_TIMEOUT (120s) fallback so the LLM call can never hang indefinitely. Applied in both LLMService and the dedicated LLM path in LLMContextSummarizer.
This commit is contained in:
@@ -2020,7 +2020,7 @@ class LLMContextSummaryRequestFrame(ControlFrame):
|
|||||||
summarization_prompt: System prompt instructing the LLM how to generate
|
summarization_prompt: System prompt instructing the LLM how to generate
|
||||||
the summary.
|
the summary.
|
||||||
summarization_timeout: Maximum time in seconds for the LLM to generate a
|
summarization_timeout: Maximum time in seconds for the LLM to generate a
|
||||||
summary. None means no timeout.
|
summary. When None, a default timeout of 120s is applied.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
request_id: str
|
request_id: str
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMe
|
|||||||
from pipecat.utils.asyncio.task_manager import BaseTaskManager
|
from pipecat.utils.asyncio.task_manager import BaseTaskManager
|
||||||
from pipecat.utils.base_object import BaseObject
|
from pipecat.utils.base_object import BaseObject
|
||||||
from pipecat.utils.context.llm_context_summarization import (
|
from pipecat.utils.context.llm_context_summarization import (
|
||||||
|
DEFAULT_SUMMARIZATION_TIMEOUT,
|
||||||
LLMContextSummarizationConfig,
|
LLMContextSummarizationConfig,
|
||||||
LLMContextSummarizationUtil,
|
LLMContextSummarizationUtil,
|
||||||
)
|
)
|
||||||
@@ -278,21 +279,20 @@ class LLMContextSummarizer(BaseObject):
|
|||||||
llm: The dedicated LLM service to use for summarization.
|
llm: The dedicated LLM service to use for summarization.
|
||||||
frame: The summarization request frame.
|
frame: The summarization request frame.
|
||||||
"""
|
"""
|
||||||
|
timeout = frame.summarization_timeout or DEFAULT_SUMMARIZATION_TIMEOUT
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if frame.summarization_timeout:
|
summary, last_index = await asyncio.wait_for(
|
||||||
summary, last_index = await asyncio.wait_for(
|
llm._generate_summary(frame),
|
||||||
llm._generate_summary(frame),
|
timeout=timeout,
|
||||||
timeout=frame.summarization_timeout,
|
)
|
||||||
)
|
|
||||||
else:
|
|
||||||
summary, last_index = await llm._generate_summary(frame)
|
|
||||||
result_frame = LLMContextSummaryResultFrame(
|
result_frame = LLMContextSummaryResultFrame(
|
||||||
request_id=frame.request_id,
|
request_id=frame.request_id,
|
||||||
summary=summary,
|
summary=summary,
|
||||||
last_summarized_index=last_index,
|
last_summarized_index=last_index,
|
||||||
)
|
)
|
||||||
except asyncio.TimeoutError:
|
except asyncio.TimeoutError:
|
||||||
error = f"Context summarization timed out after {frame.summarization_timeout}s"
|
error = f"Context summarization timed out after {timeout}s"
|
||||||
logger.error(f"{self}: {error}")
|
logger.error(f"{self}: {error}")
|
||||||
result_frame = LLMContextSummaryResultFrame(
|
result_frame = LLMContextSummaryResultFrame(
|
||||||
request_id=frame.request_id,
|
request_id=frame.request_id,
|
||||||
|
|||||||
@@ -62,6 +62,7 @@ from pipecat.services.ai_service import AIService
|
|||||||
from pipecat.services.settings import LLMSettings
|
from pipecat.services.settings import LLMSettings
|
||||||
from pipecat.turns.user_turn_completion_mixin import UserTurnCompletionLLMServiceMixin
|
from pipecat.turns.user_turn_completion_mixin import UserTurnCompletionLLMServiceMixin
|
||||||
from pipecat.utils.context.llm_context_summarization import (
|
from pipecat.utils.context.llm_context_summarization import (
|
||||||
|
DEFAULT_SUMMARIZATION_TIMEOUT,
|
||||||
LLMContextSummarizationUtil,
|
LLMContextSummarizationUtil,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -436,18 +437,15 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
|
|||||||
last_index = -1
|
last_index = -1
|
||||||
error = None
|
error = None
|
||||||
|
|
||||||
|
timeout = frame.summarization_timeout or DEFAULT_SUMMARIZATION_TIMEOUT
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if frame.summarization_timeout:
|
summary, last_index = await asyncio.wait_for(
|
||||||
summary, last_index = await asyncio.wait_for(
|
self._generate_summary(frame),
|
||||||
self._generate_summary(frame),
|
timeout=timeout,
|
||||||
timeout=frame.summarization_timeout,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
summary, last_index = await self._generate_summary(frame)
|
|
||||||
except asyncio.TimeoutError:
|
|
||||||
await self.push_error(
|
|
||||||
error_msg=f"Context summarization timed out after {frame.summarization_timeout}s"
|
|
||||||
)
|
)
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
await self.push_error(error_msg=f"Context summarization timed out after {timeout}s")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
error = f"Error generating context summary: {e}"
|
error = f"Error generating context summary: {e}"
|
||||||
await self.push_error(error, exception=e)
|
await self.push_error(error, exception=e)
|
||||||
|
|||||||
@@ -20,6 +20,9 @@ from loguru import logger
|
|||||||
|
|
||||||
from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage
|
from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage
|
||||||
|
|
||||||
|
# Fallback timeout (seconds) used when summarization_timeout is None.
|
||||||
|
DEFAULT_SUMMARIZATION_TIMEOUT = 120.0
|
||||||
|
|
||||||
# Token estimation constants
|
# Token estimation constants
|
||||||
CHARS_PER_TOKEN = 4 # Industry-standard heuristic: 1 token ≈ 4 characters
|
CHARS_PER_TOKEN = 4 # Industry-standard heuristic: 1 token ≈ 4 characters
|
||||||
TOKEN_OVERHEAD_PER_MESSAGE = 10 # Estimated structural overhead per message
|
TOKEN_OVERHEAD_PER_MESSAGE = 10 # Estimated structural overhead per message
|
||||||
@@ -89,7 +92,6 @@ class LLMContextSummarizationConfig:
|
|||||||
summarization_timeout: Maximum time in seconds to wait for the LLM to
|
summarization_timeout: Maximum time in seconds to wait for the LLM to
|
||||||
generate a summary. If the call exceeds this timeout, summarization
|
generate a summary. If the call exceeds this timeout, summarization
|
||||||
is aborted with an error and future summarizations are unblocked.
|
is aborted with an error and future summarizations are unblocked.
|
||||||
Set to None to disable the timeout.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
max_context_tokens: int = 8000
|
max_context_tokens: int = 8000
|
||||||
@@ -99,7 +101,7 @@ class LLMContextSummarizationConfig:
|
|||||||
summarization_prompt: Optional[str] = None
|
summarization_prompt: Optional[str] = None
|
||||||
summary_message_template: str = "Conversation summary: {summary}"
|
summary_message_template: str = "Conversation summary: {summary}"
|
||||||
llm: Optional["LLMService"] = None
|
llm: Optional["LLMService"] = None
|
||||||
summarization_timeout: Optional[float] = 120.0
|
summarization_timeout: float = DEFAULT_SUMMARIZATION_TIMEOUT
|
||||||
|
|
||||||
def __post_init__(self):
|
def __post_init__(self):
|
||||||
"""Validate configuration parameters."""
|
"""Validate configuration parameters."""
|
||||||
|
|||||||
Reference in New Issue
Block a user