fix: extend cancel_on_interruption=False regression fix to remaining realtime services
Applies the same async-tool message routing introduced for AWSNovaSonicLLMService
and OpenAIRealtimeLLMService to additional realtime LLM services where the
flag's intent ("keep talking while the tool runs") is achievable:
- GrokRealtimeLLMService (xAI Realtime — also benefits the deprecated Grok
alias since it re-exports the xAI module)
- AzureRealtimeLLMService picks up the fix transitively by inheriting from
OpenAIRealtimeLLMService — no code change needed.
GrokRealtimeLLMService's _process_completed_function_calls now matches
the canonical pattern: skip LLMSpecificMessage, detect async-tool messages
via parse_message and route them — started skipped silently, intermediate
logged as an error and surfaced via push_error, final delivered through
the same channel as a synchronous result.
UltravoxRealtimeLLMService instead gets a one-time warning when async-tool
messages appear in the context. The Ultravox API freezes the conversation
during tool execution
(https://docs.ultravox.ai/tools/async-tools#custom-tool-timeouts), so the
flag's "keep talking while the tool runs" intent isn't achievable there —
applying the same code pattern would mislead users into expecting a UX
Ultravox can't deliver. Surfacing a clear warning is the right behavior
until Ultravox grows true async tool support.
Adds async-tool example files for Grok and Azure modeled on the existing
Nova Sonic / OpenAI Realtime ones (10s simulated network delay, weather
tool registered with cancel_on_interruption=False).
Two services remain excluded:
- GeminiLiveLLMService — the async-tool path needs deeper investigation.
- InworldRealtimeLLMService — appears to have a pre-existing problem
with even simple synchronous tool calling on its Realtime API (the
request reaches the server fine, but response generation fails with a
generic server_error).
This commit is contained in:
Paul Kompfner
@@ -45,7 +45,8 @@ from pipecat.frames.frames import (
|
||||
UserAudioRawFrame,
|
||||
VADUserStoppedSpeakingFrame,
|
||||
)
|
||||
from pipecat.processors.aggregators.llm_context import LLMContext
|
||||
from pipecat.processors.aggregators import async_tool_messages
|
||||
from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage
|
||||
from pipecat.processors.frame_processor import FrameDirection
|
||||
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
|
||||
from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven, assert_given
|
||||
@@ -218,6 +219,7 @@ class UltravoxRealtimeLLMService(LLMService):
|
||||
self._disconnecting = False
|
||||
self._bot_responding: Literal[None, "text", "voice"] = None
|
||||
self._last_user_id: str | None = None
|
||||
self._async_tool_warning_logged: bool = False
|
||||
|
||||
self._sample_rate = 48000
|
||||
self._resampler = create_stream_resampler()
|
||||
@@ -413,6 +415,33 @@ class UltravoxRealtimeLLMService(LLMService):
|
||||
await self.push_frame(frame, direction)
|
||||
|
||||
async def _handle_context(self, context: LLMContext):
|
||||
# If the user registered a function with cancel_on_interruption=False,
|
||||
# the aggregator emits async-tool-style messages into the context. We
|
||||
# don't (currently) honor those on Ultravox: the Ultravox API freezes
|
||||
# the conversation during tool execution
|
||||
# (https://docs.ultravox.ai/tools/async-tools#custom-tool-timeouts),
|
||||
# so the "keep talking while the tool runs" intent of the flag is
|
||||
# structurally not achievable here. Surface a one-time warning so
|
||||
# users see they're not getting what they expect.
|
||||
if not self._async_tool_warning_logged:
|
||||
for message in context.get_messages():
|
||||
if isinstance(message, LLMSpecificMessage):
|
||||
continue
|
||||
if async_tool_messages.parse_message(message) is not None:
|
||||
logger.error(
|
||||
f"{self}: cancel_on_interruption=False is not supported by "
|
||||
f"Ultravox: the conversation freezes during tool execution, so "
|
||||
f"the 'keep talking while the tool runs' intent of the flag "
|
||||
f"would not be achievable anyway. Use "
|
||||
f"cancel_on_interruption=True (the default) or a non-realtime "
|
||||
f"LLM service if your tool needs the async semantics."
|
||||
)
|
||||
await self.push_error(
|
||||
error_msg="cancel_on_interruption=False is not supported by Ultravox.",
|
||||
)
|
||||
self._async_tool_warning_logged = True
|
||||
break
|
||||
|
||||
# Ultravox handles all context server-side, so the only context we may
|
||||
# need to handle here is new function call results.
|
||||
for message in reversed(context.messages):
|
||||
|
||||
@@ -47,7 +47,8 @@ from pipecat.frames.frames import (
|
||||
UserStoppedSpeakingFrame,
|
||||
)
|
||||
from pipecat.metrics.metrics import LLMTokenUsage
|
||||
from pipecat.processors.aggregators.llm_context import LLMContext
|
||||
from pipecat.processors.aggregators import async_tool_messages
|
||||
from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage
|
||||
from pipecat.processors.frame_processor import FrameDirection
|
||||
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
|
||||
from pipecat.services.settings import (
|
||||
@@ -913,6 +914,43 @@ class GrokRealtimeLLMService(LLMService[GrokRealtimeLLMAdapter]):
|
||||
sent_new_result = False
|
||||
|
||||
for message in self._context.get_messages():
|
||||
# LLMSpecificMessages are opaque provider-specific payloads, not
|
||||
# standard tool-result messages — skip them.
|
||||
if isinstance(message, LLMSpecificMessage):
|
||||
continue
|
||||
|
||||
# Async-tool messages live alongside regular tool messages in the
|
||||
# context; detect and route them before the regular logic so we
|
||||
# don't try to send the async-tool envelope JSON as a tool result.
|
||||
async_payload = async_tool_messages.parse_message(message)
|
||||
if async_payload is not None:
|
||||
if async_payload.tool_call_id in self._completed_tool_calls:
|
||||
continue
|
||||
if async_payload.kind == "started":
|
||||
# The provider already issued the tool call and natively
|
||||
# awaits a result; nothing to send for the started marker.
|
||||
continue
|
||||
if async_payload.kind == "intermediate":
|
||||
logger.error(
|
||||
f"{self}: Grok Realtime does not support streamed async "
|
||||
f"tool results; dropping intermediate result for "
|
||||
f"tool_call_id={async_payload.tool_call_id}. Use a "
|
||||
f"non-realtime LLM service if your tool needs to "
|
||||
f"stream intermediate results."
|
||||
)
|
||||
await self.push_error(
|
||||
error_msg="Grok Realtime does not support streamed async tool results.",
|
||||
)
|
||||
continue
|
||||
# kind == "final": deliver via the formal tool-result channel
|
||||
# — same path as a synchronous tool result, just delayed.
|
||||
if send_new_results:
|
||||
sent_new_result = True
|
||||
await self._send_tool_result(async_payload.tool_call_id, async_payload.result)
|
||||
self._completed_tool_calls.add(async_payload.tool_call_id)
|
||||
continue
|
||||
|
||||
# Look for newly-completed "regular" (as opposed to async-tool) results
|
||||
if message.get("role") and message.get("content") != "IN_PROGRESS":
|
||||
tool_call_id = message.get("tool_call_id")
|
||||
if tool_call_id and tool_call_id not in self._completed_tool_calls:
|
||||
@@ -939,6 +977,7 @@ class GrokRealtimeLLMService(LLMService[GrokRealtimeLLMAdapter]):
|
||||
|
||||
async def _send_tool_result(self, tool_call_id: str, result: str):
|
||||
"""Send a tool call result to Grok."""
|
||||
logger.debug(f"Sending tool result to Grok Realtime for tool_call_id={tool_call_id}")
|
||||
item = events.ConversationItem(
|
||||
type="function_call_output",
|
||||
call_id=tool_call_id,
|
||||
|
||||
Reference in New Issue
Block a user