feat(workflow): enhance message stages and image routing

This commit is contained in:
Xin Wang
2026-08-03 12:38:02 +08:00
parent b0991f239e
commit 4c43e167db
13 changed files with 593 additions and 157 deletions

View File

@@ -173,6 +173,35 @@ def _image_data_uri(frame: UserImageRawFrame) -> str:
return f"data:image/jpeg;base64,{encoded}"
def _multimodal_user_input_frame(
image_frame: UserImageRawFrame,
prompt_text: str,
) -> LLMMessagesAppendFrame:
"""Submit an explicit camera capture through the normal user-turn path.
``UserImageRawFrame`` is appended by Pipecat's assistant-side aggregator,
which pushes context upstream directly into the LLM. Workflow routing sits
on the downstream user-turn path, so queuing the raw frame would let the
Agent see the image while skipping edge evaluation. A standard multimodal
user message keeps text and image turns on the same routing path.
"""
return LLMMessagesAppendFrame(
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": prompt_text},
{
"type": "image_url",
"image_url": {"url": _image_data_uri(image_frame)},
},
],
}
],
run_llm=True,
)
async def _analyze_image_with_vision_model(
cfg: AssistantConfig,
frame: UserImageRawFrame,
@@ -783,10 +812,12 @@ async def run_pipeline(
raise ValueError("等待摄像头视频帧超时") from exc
if native_vision:
image_frame.text = value.prompt_text
image_frame.append_to_context = True
image_frame.request = None
await worker.queue_frame(image_frame)
input_frame = await asyncio.to_thread(
_multimodal_user_input_frame,
image_frame,
value.prompt_text,
)
await worker.queue_frame(input_frame)
return
try: