feat(workflow): enhance message stages and image routing
This commit is contained in:
@@ -173,6 +173,35 @@ def _image_data_uri(frame: UserImageRawFrame) -> str:
|
||||
return f"data:image/jpeg;base64,{encoded}"
|
||||
|
||||
|
||||
def _multimodal_user_input_frame(
|
||||
image_frame: UserImageRawFrame,
|
||||
prompt_text: str,
|
||||
) -> LLMMessagesAppendFrame:
|
||||
"""Submit an explicit camera capture through the normal user-turn path.
|
||||
|
||||
``UserImageRawFrame`` is appended by Pipecat's assistant-side aggregator,
|
||||
which pushes context upstream directly into the LLM. Workflow routing sits
|
||||
on the downstream user-turn path, so queuing the raw frame would let the
|
||||
Agent see the image while skipping edge evaluation. A standard multimodal
|
||||
user message keeps text and image turns on the same routing path.
|
||||
"""
|
||||
return LLMMessagesAppendFrame(
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": prompt_text},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": _image_data_uri(image_frame)},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
run_llm=True,
|
||||
)
|
||||
|
||||
|
||||
async def _analyze_image_with_vision_model(
|
||||
cfg: AssistantConfig,
|
||||
frame: UserImageRawFrame,
|
||||
@@ -783,10 +812,12 @@ async def run_pipeline(
|
||||
raise ValueError("等待摄像头视频帧超时") from exc
|
||||
|
||||
if native_vision:
|
||||
image_frame.text = value.prompt_text
|
||||
image_frame.append_to_context = True
|
||||
image_frame.request = None
|
||||
await worker.queue_frame(image_frame)
|
||||
input_frame = await asyncio.to_thread(
|
||||
_multimodal_user_input_frame,
|
||||
image_frame,
|
||||
value.prompt_text,
|
||||
)
|
||||
await worker.queue_frame(input_frame)
|
||||
return
|
||||
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user